diff --git a/Cargo.lock b/Cargo.lock index 75851af..38a6397 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -610,9 +610,9 @@ dependencies = [ [[package]] name = "ctutils" -version = "0.4.3" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03bb0e1cc970d482d121d9a1744999169b69a07470b3d644a7894e53fcaf4574" +checksum = "38d28fb6aa7dd680b3ffd1a3e64e115c177bdba63c0ae7f9cd48129ff05c4fa0" dependencies = [ "cmov", ] @@ -1528,21 +1528,6 @@ version = "0.2.190" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" -[[package]] -name = "libfs" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b79c3145dcaad30b6d7b1e30c9d2d23799b882e9a2e9d1fc7bcc0699c5279ff9" -dependencies = [ - "cfg-if", - "libc", - "linux-raw-sys", - "log", - "rustix", - "thiserror 2.0.21", - "xattr", -] - [[package]] name = "libredox" version = "0.1.25" @@ -1763,7 +1748,7 @@ dependencies = [ [[package]] name = "omnect-cli" -version = "0.28.5" +version = "0.29.0" dependencies = [ "actix-web", "anyhow", @@ -1782,7 +1767,6 @@ dependencies = [ "hmac", "httpmock", "keyring-core", - "libfs", "log", "mbrman", "num_cpus", @@ -1793,6 +1777,8 @@ dependencies = [ "regex", "reqwest 0.13.5", "ring", + "roxmltree", + "rustix", "serde", "serde_json", "serde_path_to_error", @@ -2222,6 +2208,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "roxmltree" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9da7f59328b09405b181fee1018925fad50f157afb43c0c0f5c522c256dd5907" +dependencies = [ + "memchr", +] + [[package]] name = "rustc_version" version = "0.4.1" diff --git a/Cargo.toml b/Cargo.toml index c7db19b..c1b0c5d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ license = "MIT OR Apache-2.0" name = "omnect-cli" readme = "README.md" repository = "https://github.com/omnect/omnect-cli" -version = "0.28.5" +version = "0.29.0" [dependencies] actix-web = { version = "4.17", default-features = false, features = ["macros"] } @@ -31,7 +31,6 @@ hmac = { version = "0.13", default-features = false } mbrman = { version = "0.6", default-features = false } omnect-crypto = { git = "https://github.com/omnect/omnect-crypto.git", tag = "0.5.0" } keyring-core = { version = "1.0", default-features = false } -libfs = { version = "0.9", default-features = false } log = { version = "0.4", default-features = false } num_cpus = { version = "1.17", default-features = false } oauth2 = { version = "5.0", default-features = false, features = [ @@ -44,6 +43,8 @@ reqwest = { version = "0.13", default-features = false, features = [ "json", "native-tls", ] } +roxmltree = { version = "0.21", default-features = false, features = ["std"] } +rustix = { version = "1.1", default-features = false, features = ["fs", "std"] } serde = { version = "1.0", default-features = false, features = ["derive"] } serde_json = { version = "1.0", default-features = false } serde_path_to_error = { version = "0.1", default-features = false } @@ -81,5 +82,5 @@ tar = "0.4" # metadata for building with cargo-deb (https://crates.io/crates/cargo-deb) [package.metadata.deb] -depends = "bmap-tools, e2tools, keychain, libc6 (>= 2.39), libmagic1, libssl3 (>= 3.0.0), mtools" +depends = "e2tools, keychain, libc6 (>= 2.39), libmagic1, libssl3 (>= 3.0.0), mtools" revision = "" diff --git a/Dockerfile b/Dockerfile index add2cbf..91ba58a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,6 @@ COPY ${debian_dir}/omnect-cli_${omnect_cli_version}_amd64.deb omnect-cli_${omnec RUN apt-get update && \ apt-get install -y --no-install-recommends \ - bmap-tools \ ca-certificates \ e2tools \ fdisk \ @@ -34,14 +33,11 @@ RUN < - **Note2**: The ssh tunnel option requires some additional settings. See [here](Usage-with-docker) for more details.
- **Note3**: The docker inject command is not supported by omnect-cli docker image.
. + **Note1**: The ssh tunnel option requires some additional settings. See [here](Usage-with-docker) for more details.
+ **Note2**: The docker inject command is not supported by omnect-cli docker image.
. # Build from sources @@ -184,6 +183,20 @@ omnect-cli file copy-to-image --help - File permissions: inject `systemd-tmpfiles.d` - Wifi: inject `wpa_supplicant-wlan0.conf` +## Output image + +A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression (`.xz`, `.bz2` or `.gz`), e.g. `my-image.wic.xz` for `my-image.wic`. An image that already has an extension of that compression keeps its name. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. Its name must have the extension of the `-p` compression, or no compression extension without `-p`. + +The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. The output image is written to a new file in its directory and then renamed, so the directory needs write access and space for one more image. The new file belongs to the user who runs `omnect-cli`. `copy-from-image` only reads the image and writes nothing. + +## bmap files + +Commands that change an image create a bmap file with `-b` (`--output-bmap`). It is written next to the output image as `.bmap`, e.g. `my-image.wic.bmap` for `my-image.wic` and `my-image.wic.xz.bmap` for `my-image.wic.xz`, and replaces a file with this name. `bmaptool` finds it there without `--bmap`. The command fails if this file is the bmap of an image that stays, e.g. `my-image.wic.bmap` when `-i my-image.wic.xz` writes `my-image.wic`, also with `--input-bmap` or `--no-input-bmap`. Choose another output with `-o` then. + +The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--input-bmap`, else the bmap next to the image that `bmaptool` would use: `.bmap`, else the same with one extension of the image removed at a time, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image and sha256 range checksums that match it, otherwise the command fails. A command without `-b` fails if such a bmap exists for the output image (e.g. `my-image.wic.bmap` for `my-image.wic.xz`), because `bmaptool` would use it and it would not match the image. So when you run several commands on one image, use `-b` on each of them. `--no-input-bmap` skips a bmap next to the image that does not match, and the created bmap then maps the whole image. The created bmap maps every range of the input bmap, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). It also maps every block that the image file stores as data, which includes the blocks `omnect-cli` changed. The image file itself stays sparse. + +Without an input bmap, the created bmap maps the whole image, and a flash writes every block. + ## ssh tunnel ### Inject ssh tunnel credentials diff --git a/project-context.md b/project-context.md index f1dd9ea..7d8ff67 100644 --- a/project-context.md +++ b/project-context.md @@ -50,7 +50,9 @@ - `src/docker.rs` — `docker pull --platform` + `docker save` for multi-arch images - `src/image.rs` — firmware image architecture detection (ARM32/ARM64/x86_64) - `src/file/mod.rs` — high-level image operations: identity config, certs, hostname patching -- `src/file/functions.rs` — partition read/modify/write via dd + e2cp/mcopy +- `src/file/functions.rs` — sparse partition read, modify via e2cp/mcopy, write back of changed blocks only +- `src/file/bmap.rs` — input bmap lookup and checksum check; writes the output bmap: input ranges (or the whole image) plus data blocks +- `src/file/sparse.rs` — data/hole ranges via `SEEK_DATA`/`SEEK_HOLE`, sparse range copy - `src/file/partition.rs` — GPT/MBR partition table parsing - `src/file/compression.rs` — xz/bzip2/gzip compress/decompress with auto-detection - `src/validators/` — validation for identity config (TOML), device-update config (JSON), SSH keys @@ -58,7 +60,7 @@ ## 4. Repository-Specific Constraints -- External tools `e2cp`, `e2mkdir`, `mcopy`, `mmd`, `dd`, `fallocate`, `ssh-keygen`, `fdisk` must be available at runtime (Dockerfile copies them explicitly). +- External tools `e2cp`, `e2mkdir`, `mcopy`, `mmd`, `ssh-keygen`, `fdisk` must be available at runtime (Dockerfile copies them explicitly). - Partition enum maps partition names to numbers differently for GPT vs MBR — see `file/functions.rs`. - OAuth2 callback binds to `127.0.0.1:4000` and `[::1]:4000`; container mode overrides to `0.0.0.0`. - `conf/` directory uses `.gitignore` to track only `*.template` files — actual configs are generated, never committed. diff --git a/src/cli.rs b/src/cli.rs index 746e253..dd20908 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -2,12 +2,35 @@ use crate::file::{ compression::Compression, functions::{FileCopyFromParams, FileCopyToParams, Partition}, }; -use clap::Parser; +use clap::{Args, Parser}; use std::path::PathBuf; use url::Url; const COPYRIGHT: &str = "Copyright © 2021 by conplement AG"; +#[derive(Args, Debug)] +pub struct ImageOutput { + /// optional: create a bmap of the output image + #[arg(short = 'b', long = "output-bmap")] + pub output_bmap: bool, + /// optional: bmap of the input image; default: the bmap next to the image that bmaptool would use + #[arg(long = "input-bmap", requires = "output_bmap")] + pub input_bmap: Option, + /// optional: do not use a bmap next to the image, so the bmap maps the whole image + #[arg( + long = "no-input-bmap", + requires = "output_bmap", + conflicts_with = "input_bmap" + )] + pub no_input_bmap: bool, + /// optional: write the output image to this path, with the --pack-image extension or no compression extension; default: the input image, without the compression extension, plus the --pack-image extension + #[arg(short = 'o', long = "output-image")] + pub output_image: Option, + /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') + #[arg(short = 'p', long = "pack-image", value_enum)] + pub compress_image: Option, +} + // ToDo: command completion #[derive(Parser, Debug)] #[command(after_help = COPYRIGHT)] @@ -34,12 +57,8 @@ pub enum Docker { /// destination path of the docker image in the firmware image (must end in ".tar.gz") #[clap(short = 'e', long = "dest")] dest: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, } @@ -55,12 +74,8 @@ pub enum File { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// copy files from image CopyFromImage { @@ -88,12 +103,8 @@ pub enum IdentityConfig { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// EXPERIMENTAL: set transparent gateway config.toml file and additional certificates and keys SetIotedgeGatewayConfig { @@ -112,12 +123,8 @@ pub enum IdentityConfig { /// path to device identity certificate key file #[arg(short = 'k', long = "device_identity_key")] device_identity_key: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// EXPERIMENTAL: set leaf device config.toml file and additional certificate SetIotLeafSasConfig { @@ -130,12 +137,8 @@ pub enum IdentityConfig { /// path to root ca certificate file #[arg(short = 'r', long = "root_ca")] root_ca: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// set certificates in order to support X.509 based DPS provisioning and certificate renewal via EST SetDeviceCertificate { @@ -154,12 +157,8 @@ pub enum IdentityConfig { /// period of validity in days #[arg(short = 'D', long = "days")] days: u32, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// set certificates in order to support X.509 based DPS provisioning WITHOUT certificate renewal via EST SetDeviceCertificateNoEst { @@ -172,12 +171,8 @@ pub enum IdentityConfig { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// generate and set bootstrap certificate for edge ca issuance/renewal. SetEdgeCaCertificate { @@ -196,12 +191,8 @@ pub enum IdentityConfig { /// period of validity in days #[arg(short = 'D', long = "days")] days: u32, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, } @@ -217,12 +208,8 @@ pub enum IotHubDeviceUpdate { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// import update to azure iot-hub ImportUpdate { @@ -336,12 +323,8 @@ pub enum SshConfig { /// path to public key of the ssh root ca #[arg(short = 'r', long = "root_ca")] root_ca: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// set ssh connection parameters (see diff --git a/src/file/bmap.rs b/src/file/bmap.rs new file mode 100644 index 0000000..aacc6df --- /dev/null +++ b/src/file/bmap.rs @@ -0,0 +1,562 @@ +use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, sparse}; +use anyhow::{Context, Result}; +use sha2::{Digest, Sha256}; +use std::{ + fs, + ops::{Range, RangeInclusive}, + os::unix::fs::FileExt, + path::{Path, PathBuf}, +}; + +const BMAP_EXTENSION: &str = "bmap"; +const SHA256_CHECKSUM_TYPE: &str = "sha256"; +const SHA256_HEX_LEN: usize = 64; +const BMAP_VERSION: &str = "2.0"; + +/// The block ranges of an image that a bmap file maps. +#[derive(Debug)] +pub struct Bmap { + pub block_size: u64, + pub image_size: u64, + pub ranges: Vec>, + /// The sha256 of each range, if the bmap has them. + pub sha256: Option>, +} + +impl Bmap { + pub fn read(path: &Path) -> Result { + let text = fs::read_to_string(path) + .context(format!("Bmap::read: cannot read {}", path.display()))?; + let doc = roxmltree::Document::parse(&text) + .context(format!("Bmap::read: invalid xml in {}", path.display()))?; + let root = doc.root_element(); + let child = |name: &str| { + root.children() + .find(|n| n.has_tag_name(name)) + .context(format!("Bmap::read: no {name} in {}", path.display())) + }; + let number = |name: &str| -> Result { + child(name)? + .text() + .unwrap_or_default() + .trim() + .parse() + .context(format!("Bmap::read: invalid {name} in {}", path.display())) + }; + + let block_size = number("BlockSize")?; + anyhow::ensure!(block_size > 0, "Bmap::read: BlockSize is 0"); + let image_size = number("ImageSize")?; + let range_nodes: Vec<_> = child("BlockMap")? + .children() + .filter(|n| n.has_tag_name("Range")) + .collect(); + let ranges = range_nodes + .iter() + .map(|n| parse_range(n.text().unwrap_or_default())) + .collect::>()?; + let sha256 = child("ChecksumType") + .ok() + .and_then(|n| n.text()) + .is_some_and(|t| t.trim() == SHA256_CHECKSUM_TYPE) + .then(|| { + range_nodes + .iter() + .map(|n| n.attribute("chksum").map(str::to_owned)) + .collect::>() + }) + .flatten(); + + Ok(Self { + block_size, + image_size, + ranges, + sha256, + }) + } +} + +fn parse_range(text: &str) -> Result> { + let text = text.trim(); + let (first, last) = text.split_once('-').unwrap_or((text, text)); + let parse = |s: &str| { + s.trim() + .parse::() + .context(format!("parse_range: invalid range {text}")) + }; + let (first, last) = (parse(first)?, parse(last)?); + anyhow::ensure!(first <= last, "parse_range: invalid range {text}"); + + Ok(first..=last) +} + +/// Returns `explicit`, else the bmap `bmaptool` uses for `image`: `.bmap`, else the same +/// with one extension of `image` removed at a time, whichever exists first. +pub fn find_input_bmap(image: &Path, explicit: Option<&Path>) -> Result> { + if let Some(path) = explicit { + anyhow::ensure!( + path.is_file(), + "find_input_bmap: {} not found", + path.display() + ); + return Ok(Some(path.to_path_buf())); + } + + let mut image = image.to_path_buf(); + loop { + let bmap = bmap_path(&image); + if bmap.is_file() { + return Ok(Some(bmap)); + } + if image.extension().is_none() { + return Ok(None); + } + image.set_extension(""); + } +} + +pub fn bmap_path(image: &Path) -> PathBuf { + let mut path = image.as_os_str().to_owned(); + path.push(format!(".{BMAP_EXTENSION}")); + PathBuf::from(path) +} + +/// Fails if a range of `image_file` does not match its checksum in `bmap`, so the bmap of +/// another image with the same size is not used. +pub fn verify_checksums(image_file: &Path, bmap: &Bmap) -> Result<()> { + let sha256 = bmap + .sha256 + .as_ref() + .context("verify_checksums: the bmap has no sha256 range checksums")?; + let image = fs::File::open(image_file).context(format!( + "verify_checksums: cannot open {}", + image_file.display() + ))?; + let image_len = image + .metadata() + .context("verify_checksums: cannot get image size")? + .len(); + check_layout(bmap, image_len)?; + + let mut buf = vec![0u8; COPY_CHUNK_SIZE]; + for (range, expected) in bmap.ranges.iter().zip(sha256) { + let bytes = + range.start() * bmap.block_size..((range.end() + 1) * bmap.block_size).min(image_len); + let actual = range_sha256(&image, bytes, &mut buf)?; + anyhow::ensure!( + actual.eq_ignore_ascii_case(expected.trim()), + "verify_checksums: range {range:?} of the image does not match the bmap, so it is the \ + bmap of another image or of an older version of this image" + ); + } + + Ok(()) +} + +/// Writes a bmap of `image_file` to `bmap_file`. It maps the ranges of `input`, or the whole +/// image without one, and every block that the image file stores as data. A range of `input` +/// can be all zeros and a hole in the image, and a bmap flash still has to write it, else the +/// device keeps what an earlier installation left there. +pub fn write_bmap(image_file: &Path, input: Option<&Bmap>, bmap_file: &Path) -> Result<()> { + let image = fs::File::open(image_file) + .context(format!("write_bmap: cannot open {}", image_file.display()))?; + let image_len = image + .metadata() + .context("write_bmap: cannot get image size")? + .len(); + let block_size = IMAGE_BLOCK_SIZE as u64; + + let ranges = mapped_ranges(&image, image_len, input)?; + + let mut buf = vec![0u8; COPY_CHUNK_SIZE]; + let mut block_map = String::new(); + for range in &ranges { + let bytes = range.start() * block_size..((range.end() + 1) * block_size).min(image_len); + let sha256 = range_sha256(&image, bytes, &mut buf)?; + block_map.push_str(&format!( + " {}-{} \n", + range.start(), + range.end() + )); + } + let blocks = image_len.div_ceil(block_size); + let mapped: u64 = ranges.iter().map(|r| r.end() - r.start() + 1).sum(); + + // like bmaptool, the file checksum is taken with zeros in place of itself + let zeros = "0".repeat(SHA256_HEX_LEN); + let xml = format!( + "\n\ + \n \ + {image_len} \n \ + {block_size} \n \ + {blocks} \n \ + {mapped} \n \ + {SHA256_CHECKSUM_TYPE} \n \ + {zeros} \n \ + \n{block_map} \n\ + \n" + ); + let xml = xml.replacen(&zeros, &hex(&Sha256::digest(xml.as_bytes())), 1); + + fs::write(bmap_file, xml).context(format!("write_bmap: cannot write {}", bmap_file.display())) +} + +fn mapped_ranges( + image: &fs::File, + image_len: u64, + input: Option<&Bmap>, +) -> Result>> { + let block_size = IMAGE_BLOCK_SIZE as u64; + let mut ranges: Vec> = match input { + Some(bmap) => { + check_layout(bmap, image_len)?; + bmap.ranges + .iter() + .map(|r| r.start() * bmap.block_size..(r.end() + 1) * bmap.block_size) + .collect() + } + None => vec![Range { + start: 0, + end: image_len, + }], + }; + ranges.extend(sparse::data_ranges(image, 0, image_len)?); + Ok(sparse::merge( + ranges + .into_iter() + .map(|r| { + r.start / block_size * block_size..r.end.min(image_len).next_multiple_of(block_size) + }) + .filter(|r| !r.is_empty()) + .collect(), + ) + .into_iter() + .map(|r| r.start / block_size..=r.end / block_size - 1) + .collect()) +} + +fn range_sha256(image: &fs::File, bytes: Range, buf: &mut [u8]) -> Result { + let mut hasher = Sha256::new(); + let mut pos = bytes.start; + while pos < bytes.end { + let n = usize::try_from((bytes.end - pos).min(buf.len() as u64))?; + image + .read_exact_at(&mut buf[..n], pos) + .context("range_sha256: cannot read image")?; + hasher.update(&buf[..n]); + pos += n as u64; + } + + Ok(hex(&hasher.finalize())) +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +/// Fails if `bmap` is for an image of another size or maps blocks outside the image. +fn check_layout(bmap: &Bmap, image_len: u64) -> Result<()> { + anyhow::ensure!( + bmap.image_size == image_len, + "check_layout: bmap is for an image of {} bytes, the image has {image_len}", + bmap.image_size + ); + let image_blocks = image_len.div_ceil(bmap.block_size); + if let Some(range) = bmap.ranges.iter().find(|r| *r.end() >= image_blocks) { + anyhow::bail!("check_layout: range {range:?} is outside the image"); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use crate::file::bmap::*; + + const BLOCK_SIZE: u64 = IMAGE_BLOCK_SIZE as u64; + const IMAGE_BLOCKS: u64 = 16; + const DATA_BLOCK: u64 = 0; + const ZERO_BLOCKS: RangeInclusive = 2..=3; + const UNMAPPED_BLOCK: u64 = 5; + const SMALL_BLOCK_SIZE: u64 = 512; + const PARTIAL_BLOCK_LEN: usize = 100; + const DATA_BYTE: u8 = 0x5a; + // content an earlier installation left on the device + const OLD_CONTENT_BYTE: u8 = 0xa5; + const YOCTO_BMAP: &str = "testfiles/image.wic.bmap"; + const YOCTO_IMAGE_SIZE: u64 = 30408704; + const YOCTO_MAPPED_BLOCKS: u64 = 77; + + fn bmap_xml(image_size: u64, block_map: &str) -> String { + format!( + "{image_size}\ + {BLOCK_SIZE}{block_map}" + ) + } + + fn write_input_bmap(path: &Path, image_size: u64, ranges: &[RangeInclusive]) { + let block_map: String = ranges + .iter() + .map(|r| format!("{}-{}", r.start(), r.end())) + .collect(); + fs::write(path, bmap_xml(image_size, &block_map)).expect("write bmap"); + } + + /// A sparse image with data in `DATA_BLOCK` and holes everywhere else. + fn sparse_image(dir: &Path) -> PathBuf { + let path = dir.join("image.wic"); + let image = fs::File::create(&path).expect("create image"); + image + .set_len(IMAGE_BLOCKS * BLOCK_SIZE) + .expect("set image size"); + image + .write_all_at(&[DATA_BYTE; IMAGE_BLOCK_SIZE], DATA_BLOCK * BLOCK_SIZE) + .expect("write data block"); + path + } + + fn created_bmap(image: &Path, input: Option<&Bmap>) -> Bmap { + let path = image.with_extension(BMAP_EXTENSION); + write_bmap(image, input, &path).expect("write bmap"); + Bmap::read(&path).expect("read bmap") + } + + fn is_mapped(bmap: &Bmap, block: u64) -> bool { + bmap.ranges.iter().any(|r| r.contains(&block)) + } + + /// Writes the blocks `bmap` maps onto a device that still holds old content, as a bmap + /// flash does. + fn bmap_flash(image: &Path, bmap: &Bmap, device: &Path) { + let image = fs::read(image).expect("read image"); + let mut content = vec![OLD_CONTENT_BYTE; image.len()]; + for range in &bmap.ranges { + let start = usize::try_from(range.start() * bmap.block_size).expect("start"); + let end = usize::try_from((range.end() + 1) * bmap.block_size).expect("end"); + let end = end.min(image.len()); + content[start..end].copy_from_slice(&image[start..end]); + } + fs::write(device, content).expect("write device"); + } + + #[test] + fn read_parses_yocto_bmap() { + let bmap = Bmap::read(Path::new(YOCTO_BMAP)).expect("read bmap"); + + assert_eq!(bmap.block_size, BLOCK_SIZE); + assert_eq!(bmap.image_size, YOCTO_IMAGE_SIZE); + let mapped: u64 = bmap.ranges.iter().map(|r| r.end() - r.start() + 1).sum(); + assert_eq!(mapped, YOCTO_MAPPED_BLOCKS); + assert_eq!( + bmap.sha256.map(|c| c.len()), + Some(bmap.ranges.len()), + "one checksum per range" + ); + } + + #[test] + fn read_parses_single_blocks_and_spans() { + let dir = tempfile::tempdir().expect("create temp dir"); + let path = dir.path().join("image.wic.bmap"); + let block_map = " 3 5-7 "; + fs::write(&path, bmap_xml(BLOCK_SIZE, block_map)).expect("write bmap"); + + let bmap = Bmap::read(&path).expect("read bmap"); + assert_eq!(bmap.ranges, vec![3..=3, 5..=7]); + } + + #[test] + fn read_has_no_checksums_without_sha256_checksum_type() { + let dir = tempfile::tempdir().expect("create temp dir"); + let path = dir.path().join("image.wic.bmap"); + fs::write(&path, bmap_xml(BLOCK_SIZE, "0")).expect("write bmap"); + + assert_eq!(Bmap::read(&path).expect("read bmap").sha256, None); + } + + #[test] + fn read_rejects_invalid_bmaps() { + let dir = tempfile::tempdir().expect("create temp dir"); + let path = dir.path().join("image.wic.bmap"); + let invalid = [ + "1".to_string(), + bmap_xml(BLOCK_SIZE, "7-5"), + bmap_xml(BLOCK_SIZE, "x"), + "".to_string(), + ]; + + for xml in invalid { + fs::write(&path, &xml).expect("write bmap"); + assert!(Bmap::read(&path).is_err(), "accepted {xml}"); + } + } + + #[test] + fn find_input_bmap_takes_explicit_path() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic"); + let explicit = dir.path().join("other.bmap"); + + assert!(find_input_bmap(&image, Some(&explicit)).is_err()); + fs::write(&explicit, "").expect("write bmap"); + assert_eq!( + find_input_bmap(&image, Some(&explicit)).expect("find bmap"), + Some(explicit) + ); + } + + #[test] + fn find_input_bmap_looks_next_to_the_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic.xz"); + let uncompressed_bmap = dir.path().join("image.wic.bmap"); + let compressed_bmap = dir.path().join("image.wic.xz.bmap"); + + assert_eq!(find_input_bmap(&image, None).expect("find bmap"), None); + fs::write(&uncompressed_bmap, "").expect("write bmap"); + assert_eq!( + find_input_bmap(&image, None).expect("find bmap"), + Some(uncompressed_bmap) + ); + fs::write(&compressed_bmap, "").expect("write bmap"); + assert_eq!( + find_input_bmap(&image, None).expect("find bmap"), + Some(compressed_bmap) + ); + } + + #[test] + fn find_input_bmap_strips_extensions_like_bmaptool() { + let dir = tempfile::tempdir().expect("create temp dir"); + let bmap = dir.path().join("image.bmap"); + fs::write(&bmap, "").expect("write bmap"); + + for image in ["image.wic", "image.wic.xz"] { + assert_eq!( + find_input_bmap(&dir.path().join(image), None).expect("find bmap"), + Some(bmap.clone()) + ); + } + } + + #[test] + fn write_bmap_maps_zero_ranges_of_the_input_bmap() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = sparse_image(dir.path()); + let input_bmap = dir.path().join("input.bmap"); + write_input_bmap(&input_bmap, IMAGE_BLOCKS * BLOCK_SIZE, &[ZERO_BLOCKS]); + let input = Bmap::read(&input_bmap).expect("read bmap"); + + let bmap = created_bmap(&image, Some(&input)); + assert!(ZERO_BLOCKS.clone().all(|b| is_mapped(&bmap, b))); + assert!( + is_mapped(&bmap, DATA_BLOCK), + "data outside the input ranges" + ); + assert!(!is_mapped(&bmap, UNMAPPED_BLOCK)); + let file = fs::File::open(&image).expect("open image"); + assert_eq!( + sparse::data_ranges(&file, 0, IMAGE_BLOCKS * BLOCK_SIZE).expect("data ranges"), + vec![DATA_BLOCK * BLOCK_SIZE..(DATA_BLOCK + 1) * BLOCK_SIZE], + "image is not sparse any more" + ); + + let device = dir.path().join("device"); + bmap_flash(&image, &bmap, &device); + let device = fs::read(&device).expect("read device"); + let zero_start = usize::try_from(ZERO_BLOCKS.start() * BLOCK_SIZE).expect("start"); + let zero_end = usize::try_from((ZERO_BLOCKS.end() + 1) * BLOCK_SIZE).expect("end"); + assert!(device[zero_start..zero_end].iter().all(|b| *b == 0)); + } + + #[test] + fn write_bmap_without_input_maps_whole_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = sparse_image(dir.path()); + + assert_eq!( + created_bmap(&image, None).ranges, + vec![0..=IMAGE_BLOCKS - 1] + ); + } + + #[test] + fn write_bmap_maps_whole_blocks_for_a_smaller_bmap_block_size() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = sparse_image(dir.path()); + // starts in DATA_BLOCK and ends in the hole after it + let first = (DATA_BLOCK + 1) * BLOCK_SIZE / SMALL_BLOCK_SIZE - 1; + let input = Bmap { + block_size: SMALL_BLOCK_SIZE, + image_size: IMAGE_BLOCKS * BLOCK_SIZE, + ranges: vec![RangeInclusive::new(first, first + 2)], + sha256: None, + }; + + assert!(is_mapped( + &created_bmap(&image, Some(&input)), + DATA_BLOCK + 1 + )); + } + + #[test] + fn verify_checksums_accepts_bmap_of_the_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = sparse_image(dir.path()); + // a mapped last block that is shorter than the block size + fs::OpenOptions::new() + .write(true) + .open(&image) + .expect("open image") + .write_all_at(&[DATA_BYTE; PARTIAL_BLOCK_LEN], IMAGE_BLOCKS * BLOCK_SIZE) + .expect("write partial block"); + + verify_checksums(&image, &created_bmap(&image, None)).expect("verify checksums"); + } + + #[test] + fn verify_checksums_rejects_changed_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = sparse_image(dir.path()); + let bmap = created_bmap(&image, None); + fs::OpenOptions::new() + .write(true) + .open(&image) + .expect("open image") + .write_all_at(&[!DATA_BYTE], DATA_BLOCK * BLOCK_SIZE) + .expect("change data block"); + + let err = verify_checksums(&image, &bmap).expect_err("changed image accepted"); + assert!(err.to_string().contains("does not match the bmap"), "{err}"); + } + + #[test] + fn verify_checksums_rejects_bmap_without_checksums() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = sparse_image(dir.path()); + let mut bmap = created_bmap(&image, None); + bmap.sha256 = None; + + assert!(verify_checksums(&image, &bmap).is_err()); + } + + #[test] + fn bmap_of_another_image_is_rejected() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = sparse_image(dir.path()); + let bmap_file = dir.path().join("out.bmap"); + let mut other_size = created_bmap(&image, None); + other_size.image_size += BLOCK_SIZE; + let outside = Bmap { + block_size: BLOCK_SIZE, + image_size: IMAGE_BLOCKS * BLOCK_SIZE, + ranges: vec![RangeInclusive::new(IMAGE_BLOCKS, IMAGE_BLOCKS)], + sha256: Some(vec![String::new()]), + }; + + for bmap in [&other_size, &outside] { + assert!(verify_checksums(&image, bmap).is_err()); + assert!(write_bmap(&image, Some(bmap), &bmap_file).is_err()); + } + } +} diff --git a/src/file/compression.rs b/src/file/compression.rs index 16f1862..d15c1c3 100644 --- a/src/file/compression.rs +++ b/src/file/compression.rs @@ -1,10 +1,15 @@ +use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, is_zero}; use anyhow::{Context, Result}; use filemagic::Magic; use log::debug; -use std::env; -use std::fs::File; -use std::path::PathBuf; -use std::str::FromStr; +use std::{ + env, + fs::File, + io::{BufWriter, IntoInnerError, Read, Seek, SeekFrom, Write}, + os::unix::fs::FileExt, + path::{Path, PathBuf}, + str::FromStr, +}; use strum::IntoEnumIterator; use strum_macros::EnumIter; @@ -46,15 +51,17 @@ impl Compression { source: &mut std::fs::File, destination: &mut std::fs::File, ) -> std::io::Result { - let mut enc: Box = match &self { - Compression::bzip2 => Box::new(bzip2::write::BzEncoder::new( - destination, - bzip2::Compression::best(), - )), - Compression::gzip => Box::new(flate2::write::GzEncoder::new( - destination, - flate2::Compression::best(), - )), + // only `finish` reports an error while writing the trailer + let (bytes_read, _) = match &self { + Compression::bzip2 => { + let mut enc = bzip2::write::BzEncoder::new(destination, bzip2::Compression::best()); + (std::io::copy(source, &mut enc)?, enc.finish()?) + } + Compression::gzip => { + let mut enc = + flate2::write::GzEncoder::new(destination, flate2::Compression::best()); + (std::io::copy(source, &mut enc)?, enc.finish()?) + } Compression::xz { compression_level: level, } => { @@ -62,13 +69,11 @@ impl Compression { .threads(num_cpus::get() as u32) .preset(*level) .encoder()?; - Box::new(xz2::write::XzEncoder::new_stream(destination, stream)) + let mut enc = xz2::write::XzEncoder::new_stream(destination, stream); + (std::io::copy(source, &mut enc)?, enc.finish()?) } }; - - let bytes_written = std::io::copy(source, &mut enc)?; - enc.flush()?; - Ok(bytes_written) + Ok(bytes_read) } pub fn decompress( @@ -76,15 +81,18 @@ impl Compression { source: &mut std::fs::File, destination: &mut std::fs::File, ) -> std::io::Result { - let mut dec: Box = match &self { - Compression::bzip2 => Box::new(bzip2::write::BzDecoder::new(destination)), - Compression::gzip => Box::new(flate2::write::GzDecoder::new(destination)), - Compression::xz { .. } => Box::new(xz2::write::XzDecoder::new(destination)), + // the multi decoders also read concatenated streams, as pbzip2 writes them + let mut decoder: Box = match &self { + Compression::bzip2 => Box::new(bzip2::read::MultiBzDecoder::new(source)), + Compression::gzip => Box::new(flate2::read::MultiGzDecoder::new(source)), + Compression::xz { .. } => Box::new(xz2::read::XzDecoder::new_multi_decoder(source)), }; - - let bytes_written = std::io::copy(source, &mut dec)?; - dec.write_all(&[])?; - dec.flush()?; + let mut sparse = BufWriter::with_capacity(COPY_CHUNK_SIZE, SparseWriter::new(destination)?); + let bytes_written = std::io::copy(&mut decoder, &mut sparse)?; + sparse + .into_inner() + .map_err(IntoInnerError::into_error)? + .finish()?; Ok(bytes_written) } @@ -97,13 +105,35 @@ impl Compression { } fn extension(&self) -> &'static str { + self.extensions()[0] + } + + /// The first one is used for a compressed image that omnect-cli creates. + fn extensions(&self) -> &'static [&'static str] { match &self { - Compression::bzip2 => "bzip2", - Compression::gzip => "gzip", - Compression::xz { .. } => "xz", + Compression::bzip2 => &["bz2", "bzip2"], + Compression::gzip => &["gz", "gzip"], + Compression::xz { .. } => &["xz"], } } + /// `path` with the extension omnect-cli gives a compressed image appended. + pub fn compressed_path(&self, path: &Path) -> PathBuf { + let mut compressed = path.as_os_str().to_owned(); + compressed.push(format!(".{}", self.extension())); + PathBuf::from(compressed) + } + + pub fn matches_extension(&self, path: &Path) -> bool { + path.extension() + .and_then(|e| e.to_str()) + .is_some_and(|e| self.extensions().contains(&e)) + } + + pub fn has_extension(path: &Path) -> bool { + Compression::iter().any(|c| c.matches_extension(path)) + } + pub fn from_file(image_file_name: &PathBuf) -> Result> { let detector = Magic::open(Default::default()) .context("image::compression: failed to open libmagic")?; @@ -126,34 +156,313 @@ impl Compression { } } -pub fn decompress(image_file_name: &PathBuf, compression: &Compression) -> Result { - let mut new_image_file = PathBuf::from(image_file_name); +/// Skips zero blocks, so they stay holes in the file. +struct SparseWriter<'a> { + file: &'a mut File, + pos: u64, +} + +impl<'a> SparseWriter<'a> { + fn new(file: &'a mut File) -> std::io::Result { + let pos = file.stream_position()?; + // skipped zero blocks must read back as zeros + file.set_len(pos)?; + Ok(Self { file, pos }) + } + + fn finish(self) -> std::io::Result<()> { + // extends the file over zero blocks at the end + self.file.set_len(self.pos)?; + self.file.seek(SeekFrom::Start(self.pos))?; + Ok(()) + } +} + +impl Write for SparseWriter<'_> { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let block_offset = (self.pos % IMAGE_BLOCK_SIZE as u64) as usize; - if new_image_file - .extension() - .is_some_and(|ext| ext == compression.extension()) - { - new_image_file.set_extension(""); + // one call handles a run of blocks that are all zero or all data + let mut end = buf.len().min(IMAGE_BLOCK_SIZE - block_offset); + let zero = is_zero(&buf[..end]); + while end < buf.len() { + let next = buf.len().min(end + IMAGE_BLOCK_SIZE); + if is_zero(&buf[end..next]) != zero { + break; + } + end = next; + } + + let written = if zero { + end + } else { + self.file.write_at(&buf[..end], self.pos)? + }; + self.pos += written as u64; + Ok(written) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.file.flush() } +} + +/// Decompresses `image_file_name` into `dest_dir`, named without the compression extension. +pub fn decompress( + image_file_name: &Path, + compression: &Compression, + dest_dir: &Path, +) -> Result { + anyhow::ensure!( + compression.matches_extension(image_file_name), + "decompress: {} has no {:?} file extension", + image_file_name.display(), + compression.extensions() + ); + let new_image_file = dest_dir.join( + image_file_name + .file_stem() + .context("decompress: cannot get image file name")?, + ); - let mut destination = File::create(&new_image_file)?; - let mut source = File::open(image_file_name)?; + let mut source = File::open(image_file_name).context(format!( + "decompress: cannot open {}", + image_file_name.display() + ))?; + let mut destination = File::create(&new_image_file).context(format!( + "decompress: cannot create {}", + new_image_file.display() + ))?; debug!("decompress {image_file_name:?} to {new_image_file:?}"); let bytes_written = compression.decompress(&mut source, &mut destination)?; debug!("image::decompress: copied {} bytes.", bytes_written); Ok(new_image_file) } -pub fn compress(image_file_name: &PathBuf, compression: &Compression) -> Result { - let new_image_file = PathBuf::from(format!( - "{}.{}", - image_file_name.to_str().unwrap(), - compression.extension() - )); - let mut destination = File::create(&new_image_file)?; - let mut source = File::open(image_file_name)?; - debug!("compress {image_file_name:?} to {new_image_file:?}"); - let bytes_written = compression.compress(&mut source, &mut destination)?; +pub fn compress( + image_file_name: &Path, + compression: &Compression, + destination: &mut File, +) -> Result<()> { + let mut source = File::open(image_file_name).context(format!( + "compress: cannot open {}", + image_file_name.display() + ))?; + debug!("compress {image_file_name:?}"); + let bytes_written = compression.compress(&mut source, destination)?; debug!("image::compress: copied {} bytes.", bytes_written); - Ok(new_image_file) + Ok(()) +} + +#[cfg(test)] +mod tests { + use crate::file::compression::*; + use std::{fs, io::Read, os::unix::fs::MetadataExt}; + + // unit of `st_blocks` + const STAT_BLOCK_SIZE: u64 = 512; + const DATA_OFFSET: usize = 1024 * 1024 + 100; + const DATA: &[u8] = b"abc"; + const IMAGE_LEN: usize = 3 * 1024 * 1024 + 123; + const XZ_TEST_LEVEL: u32 = 1; + // cuts into the trailer, which every format checks only at the end of the stream + const TRUNCATED_BYTES: u64 = 4; + const OLD_CONTENT_BYTE: u8 = 0xa5; + const OLD_CONTENT_EXTRA_LEN: usize = 4096; + + fn all_compressions() -> [Compression; 3] { + [ + Compression::bzip2, + Compression::gzip, + Compression::xz { + compression_level: XZ_TEST_LEVEL, + }, + ] + } + + fn test_image() -> Vec { + let mut image = vec![0u8; IMAGE_LEN]; + image[DATA_OFFSET..DATA_OFFSET + DATA.len()].copy_from_slice(DATA); + image + } + + fn compress_image(compression: &Compression, image: &[u8]) -> File { + let mut source = tempfile::tempfile().expect("create source"); + source.write_all(image).expect("write source"); + source.rewind().expect("rewind source"); + + let mut compressed = tempfile::tempfile().expect("create compressed"); + compression + .compress(&mut source, &mut compressed) + .expect("compress"); + compressed.rewind().expect("rewind compressed"); + compressed + } + + fn read_all(file: &mut File) -> Vec { + let mut content = Vec::new(); + file.rewind().expect("rewind"); + file.read_to_end(&mut content).expect("read"); + content + } + + #[test] + fn decompress_keeps_zero_blocks_as_holes() { + let image = test_image(); + + for compression in all_compressions() { + let mut compressed = compress_image(&compression, &image); + let mut decompressed = tempfile::tempfile().expect("create decompressed"); + compression + .decompress(&mut compressed, &mut decompressed) + .expect("decompress"); + + assert!( + read_all(&mut decompressed) == image, + "{compression:?}: content differs" + ); + let meta = decompressed.metadata().expect("metadata"); + assert!( + meta.blocks() * STAT_BLOCK_SIZE < meta.len(), + "{compression:?}: no holes" + ); + } + } + + #[test] + fn decompress_replaces_old_content() { + let image = test_image(); + + for compression in all_compressions() { + let mut compressed = compress_image(&compression, &image); + let mut decompressed = tempfile::tempfile().expect("create decompressed"); + decompressed + .write_all(&vec![OLD_CONTENT_BYTE; IMAGE_LEN + OLD_CONTENT_EXTRA_LEN]) + .expect("write old content"); + decompressed.rewind().expect("rewind decompressed"); + + compression + .decompress(&mut compressed, &mut decompressed) + .expect("decompress"); + + assert!( + read_all(&mut decompressed) == image, + "{compression:?}: content differs" + ); + } + } + + #[test] + fn decompress_rejects_truncated_input() { + let image = test_image(); + + for compression in all_compressions() { + let mut compressed = compress_image(&compression, &image); + let len = compressed.metadata().expect("metadata").len(); + compressed + .set_len(len - TRUNCATED_BYTES) + .expect("truncate compressed"); + + let mut decompressed = tempfile::tempfile().expect("create decompressed"); + assert!( + compression + .decompress(&mut compressed, &mut decompressed) + .is_err(), + "{compression:?}: truncated input accepted" + ); + } + } + + #[test] + fn decompress_reads_concatenated_streams() { + let image = test_image(); + let (first, second) = image.split_at(DATA_OFFSET); + + for compression in all_compressions() { + let mut concatenated = tempfile::tempfile().expect("create concatenated"); + for part in [first, second] { + let mut compressed = compress_image(&compression, part); + concatenated + .write_all(&read_all(&mut compressed)) + .expect("write stream"); + } + concatenated.rewind().expect("rewind concatenated"); + + let mut decompressed = tempfile::tempfile().expect("create decompressed"); + compression + .decompress(&mut concatenated, &mut decompressed) + .expect("decompress"); + assert!( + read_all(&mut decompressed) == image, + "{compression:?}: content differs" + ); + } + } + + #[test] + fn decompress_file_strips_every_extension_of_the_compression() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = test_image(); + + for compression in all_compressions() { + for extension in compression.extensions() { + let path = dir.path().join(format!("image.wic.{extension}")); + let mut compressed = compress_image(&compression, &image); + fs::write(&path, read_all(&mut compressed)).expect("write compressed"); + + let decompressed = decompress(&path, &compression, dir.path()).expect("decompress"); + assert_eq!(decompressed, dir.path().join("image.wic")); + assert!( + fs::read(&decompressed).expect("read decompressed") == image, + "{compression:?}: content of .{extension} differs" + ); + assert!(Compression::has_extension(&path)); + } + } + } + + #[test] + fn compressed_path_has_an_extension_bmaptool_reads() { + let image = Path::new("image.wic"); + let expected = ["image.wic.bz2", "image.wic.gz", "image.wic.xz"]; + + for (compression, expected) in all_compressions().iter().zip(expected) { + assert_eq!(compression.compressed_path(image), Path::new(expected)); + } + } + + #[test] + fn decompress_file_rejects_unknown_extension() { + let dir = tempfile::tempdir().expect("create temp dir"); + let compression = Compression::gzip; + let path = dir.path().join("image.wic"); + let mut compressed = compress_image(&compression, &test_image()); + let content = read_all(&mut compressed); + fs::write(&path, &content).expect("write compressed"); + + assert!(decompress(&path, &compression, dir.path()).is_err()); + assert!( + fs::read(&path).expect("read compressed") == content, + "source was changed" + ); + } + + #[test] + fn compress_fails_when_the_trailer_cannot_be_written() { + let dir = tempfile::tempdir().expect("create temp dir"); + // an empty source makes `finish` the first write to the destination + let source = dir.path().join("empty.wic"); + fs::write(&source, []).expect("write source"); + for compression in all_compressions() { + let mut source = fs::File::open(&source).expect("open source"); + let mut full = fs::OpenOptions::new() + .write(true) + .open("/dev/full") + .expect("open /dev/full"); + assert!( + compression.compress(&mut source, &mut full).is_err(), + "{compression:?}: write error was ignored" + ); + } + } } diff --git a/src/file/functions.rs b/src/file/functions.rs index 4b8cf95..5e4a279 100644 --- a/src/file/functions.rs +++ b/src/file/functions.rs @@ -1,11 +1,16 @@ +use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, partition::SECTOR_SIZE, sparse}; use anyhow::{Context, Result}; use log::{debug, warn}; -use std::collections::HashMap; -use std::fmt::{self, Display}; -use std::fs; -use std::path::{Path, PathBuf}; -use std::process::Command; -use std::str::FromStr; +use std::{ + collections::HashMap, + fmt::{self, Display}, + fs, + ops::Range, + os::unix::fs::FileExt, + path::{Path, PathBuf}, + process::Command, + str::FromStr, +}; use stdext::function_name; use uuid::Uuid; @@ -184,12 +189,8 @@ macro_rules! try_exec_cmd { } pub fn copy_to_image(file_copy_params: &[FileCopyToParams], image_file: &Path) -> Result<()> { - // we use the folder the image is located in - // the caller is responsible to create a /tmp/ directory if needed - let working_dir = image_file - .parent() - .context("copy_to_image: cannot get directory of image")? - .to_path_buf(); + // the partition files get their own dir, so they never collide with the image + let working_dir = tempfile::tempdir().context("copy_to_image: cannot create a temp dir")?; let image_file = image_file.to_str().unwrap(); let mut partition_map: HashMap<&Partition, Vec<(&PathBuf, &PathBuf)>> = HashMap::new(); @@ -204,7 +205,7 @@ pub fn copy_to_image(file_copy_params: &[FileCopyToParams], image_file: &Path) - // 1. for each involved partition for partition in partition_map.keys() { - let mut partition_file = working_dir.clone(); + let mut partition_file = working_dir.path().to_path_buf(); let partition_info = get_partition_info(image_file, partition)?; partition_file.push(Path::new(&format!("{}.img", partition_info.num))); @@ -266,16 +267,12 @@ pub fn copy_to_image(file_copy_params: &[FileCopyToParams], image_file: &Path) - } pub fn copy_from_image(file_copy_params: &[FileCopyFromParams], image_file: &Path) -> Result<()> { - // we use the folder the image is located in - // the caller is responsible to create a /tmp/ directory if needed - let working_dir = image_file - .parent() - .context("copy_to_image: cannot get directory of image")? - .to_path_buf(); + // the partition files get their own dir, so they never collide with the image + let working_dir = tempfile::tempdir().context("copy_from_image: cannot create a temp dir")?; let image_file = image_file.to_str().unwrap(); for param in file_copy_params.iter() { - let mut partition_file = working_dir.clone(); + let mut partition_file = working_dir.path().to_path_buf(); let partition_info = get_partition_info(image_file, ¶m.partition)?; let in_file = param.in_file.to_str().unwrap(); @@ -297,7 +294,7 @@ pub fn copy_from_image(file_copy_params: &[FileCopyFromParams], image_file: &Pat // copy if param.partition == Partition::boot { - let mut tmp_out_file = working_dir.clone(); + let mut tmp_out_file = working_dir.path().to_path_buf(); // mcopy deadlocks when target file is not residing in workingdir so we copy to a temp file tmp_out_file.push(format!( "{}-{}", @@ -412,20 +409,15 @@ fn read_partition( return Ok(()); } - let mut dd = Command::new("dd"); - dd.arg(format!("if={image_file}")) - .arg(format!("of={partition_file}")) - .arg("bs=512") - .arg(format!("skip={}", partition_info.start)) - .arg(format!("count={}", partition_info.count)) - .arg("conv=sparse") - .arg("status=none"); - exec_cmd!(dd); + let image = + fs::File::open(image_file).context(format!("read_partition: cannot open {image_file}"))?; - let mut sync = Command::new("sync"); - exec_cmd!(sync); - - Ok(()) + sparse::copy_to_new_file( + &image, + partition_info.start * SECTOR_SIZE, + Path::new(partition_file), + partition_info.count * SECTOR_SIZE, + ) } fn write_partition( @@ -433,34 +425,268 @@ fn write_partition( partition_file: &str, partition_info: &PartitionInfo, ) -> Result<()> { - let mut dd = Command::new("dd"); - dd.arg(format!("if={partition_file}")) - .arg(format!("of={image_file}")) - .arg("bs=512") - .arg(format!("seek={}", partition_info.start)) - .arg(format!("count={}", partition_info.count)) - .arg("conv=notrunc,sparse") - .arg("status=none"); - exec_cmd!(dd); - - let mut fallocate = Command::new("fallocate"); - fallocate.arg("-d").arg(image_file); - exec_cmd!(fallocate); - - let mut sync = Command::new("sync"); - exec_cmd!(sync); + let partition = fs::File::open(partition_file) + .context(format!("write_partition: cannot open {partition_file}"))?; + let image = fs::OpenOptions::new() + .read(true) + .write(true) + .open(image_file) + .context(format!("write_partition: cannot open {image_file}"))?; + + let image_offset = partition_info.start * SECTOR_SIZE; + let len = partition + .metadata() + .context("write_partition: cannot get partition file size")? + .len() + .min(partition_info.count * SECTOR_SIZE); + + // Only write blocks that differ: unchanged blocks keep their allocation, so a bmap + // created from the result still maps every block the source image mapped. A zero block + // that a tool wrote into a hole is data in the partition file and must become data in the + // image too, else a bmap flash keeps the old content there. + let partition_data = sparse::data_ranges(&partition, 0, len)?; + let image_data = sparse::data_ranges(&image, image_offset, image_offset + len)?; + // a block that is a hole in both files is zero in both, so it never needs a write + let block_size = IMAGE_BLOCK_SIZE as u64; + let spans = sparse::merge( + partition_data + .iter() + .cloned() + .chain( + image_data + .iter() + .map(|r| r.start - image_offset..r.end - image_offset), + ) + .map(|r| r.start / block_size * block_size..r.end.next_multiple_of(block_size).min(len)) + .collect(), + ); + let mut src = vec![0u8; COPY_CHUNK_SIZE]; + let mut dst = vec![0u8; COPY_CHUNK_SIZE]; + for span in spans { + write_changed_blocks( + &partition, + &image, + image_offset, + span, + (&partition_data, &image_data), + (&mut src, &mut dst), + )?; + } Ok(()) } -pub fn generate_bmap_file(image_file: &str) -> Result<()> { - let mut bmaptool = Command::new("bmaptool"); - bmaptool - .arg("create") - .arg("-o") - .arg(format!("{image_file}.bmap")) - .arg(image_file); - exec_cmd!(bmaptool); +fn write_changed_blocks( + partition: &fs::File, + image: &fs::File, + image_offset: u64, + span: Range, + (partition_data, image_data): (&[Range], &[Range]), + (src, dst): (&mut [u8], &mut [u8]), +) -> Result<()> { + let mut pos = span.start; + while pos < span.end { + let n = usize::try_from((span.end - pos).min(COPY_CHUNK_SIZE as u64))?; + partition + .read_exact_at(&mut src[..n], pos) + .context("write_partition: cannot read partition file")?; + image + .read_exact_at(&mut dst[..n], image_offset + pos) + .context("write_partition: cannot read image")?; + + let blocks = src[..n] + .chunks(IMAGE_BLOCK_SIZE) + .zip(dst[..n].chunks(IMAGE_BLOCK_SIZE)); + for (i, (s, d)) in blocks.enumerate() { + let block_pos = pos + (i * IMAGE_BLOCK_SIZE) as u64; + let block = block_pos..block_pos + s.len() as u64; + let image_block = image_offset + block.start..image_offset + block.end; + if s != d || (has_data(partition_data, &block) && has_hole(image_data, &image_block)) { + image + .write_all_at(s, image_block.start) + .context("write_partition: cannot write image")?; + } + } + pos += n as u64; + } Ok(()) } + +fn has_data(data_ranges: &[Range], block: &Range) -> bool { + let i = data_ranges.partition_point(|r| r.end <= block.start); + data_ranges.get(i).is_some_and(|r| r.start < block.end) +} + +fn has_hole(data_ranges: &[Range], block: &Range) -> bool { + let i = data_ranges.partition_point(|r| r.end <= block.start); + !data_ranges + .get(i) + .is_some_and(|r| r.start <= block.start && block.end <= r.end) +} + +#[cfg(test)] +mod tests { + use crate::file::functions::*; + use std::os::unix::fs::MetadataExt; + + const PART_NUM: u32 = 1; + const PART_START: u64 = 8; + const PART_SECTORS: u64 = 64; + const OLD_CONTENT_BYTE: u8 = 0xa5; + const ZEROED_BLOCK: usize = 2; + // the first partition of an old MBR layout + const UNALIGNED_PART_START: u64 = 63; + + fn partition_info() -> PartitionInfo { + PartitionInfo { + num: PART_NUM, + start: PART_START, + count: PART_SECTORS, + } + } + + #[test] + fn read_partition_fails_for_partition_past_the_image_end() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic"); + let partition = dir.path().join("partition.img"); + fs::File::create(&image) + .expect("create image") + .set_len((PART_START + PART_SECTORS - 1) * SECTOR_SIZE) + .expect("set image size"); + + assert!( + read_partition( + image.to_str().expect("image path"), + partition.to_str().expect("partition path"), + &partition_info(), + ) + .is_err() + ); + } + + #[test] + fn write_partition_writes_block_changed_to_zeros() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic"); + let partition = dir.path().join("partition.img"); + let image_len = usize::try_from((PART_START + PART_SECTORS) * SECTOR_SIZE).expect("size"); + let part_offset = usize::try_from(PART_START * SECTOR_SIZE).expect("offset"); + + fs::write(&image, vec![OLD_CONTENT_BYTE; image_len]).expect("write image"); + let mut content = vec![OLD_CONTENT_BYTE; image_len - part_offset]; + content[ZEROED_BLOCK * IMAGE_BLOCK_SIZE..(ZEROED_BLOCK + 1) * IMAGE_BLOCK_SIZE].fill(0); + fs::write(&partition, &content).expect("write partition"); + + write_partition( + image.to_str().expect("image path"), + partition.to_str().expect("partition path"), + &partition_info(), + ) + .expect("write partition"); + + let written = fs::read(&image).expect("read image"); + assert!(written[part_offset..] == content[..], "partition differs"); + } + + #[test] + fn write_partition_keeps_holes_of_unchanged_blocks() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic"); + let partition = dir.path().join("partition.img"); + fs::File::create(&image) + .expect("create image") + .set_len((PART_START + PART_SECTORS) * SECTOR_SIZE) + .expect("set image size"); + fs::File::create(&partition) + .expect("create partition") + .set_len(PART_SECTORS * SECTOR_SIZE) + .expect("set partition size"); + let allocated = || fs::metadata(&image).expect("image metadata").blocks(); + let before = allocated(); + + write_partition( + image.to_str().expect("image path"), + partition.to_str().expect("partition path"), + &partition_info(), + ) + .expect("write partition"); + + assert_eq!(allocated(), before, "unchanged zero blocks were written"); + } + + #[test] + fn write_partition_maps_zero_block_of_an_unaligned_partition() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic"); + let partition = dir.path().join("partition.img"); + let info = PartitionInfo { + num: PART_NUM, + start: UNALIGNED_PART_START, + count: PART_SECTORS, + }; + let image_file = fs::File::create(&image).expect("create image"); + image_file + .set_len((UNALIGNED_PART_START + PART_SECTORS) * SECTOR_SIZE) + .expect("set image size"); + let partition_file = fs::File::create(&partition).expect("create partition"); + partition_file + .set_len(PART_SECTORS * SECTOR_SIZE) + .expect("set partition size"); + let block = (ZEROED_BLOCK * IMAGE_BLOCK_SIZE) as u64; + partition_file + .write_all_at(&[0; IMAGE_BLOCK_SIZE], block) + .expect("write zero block"); + + write_partition( + image.to_str().expect("image path"), + partition.to_str().expect("partition path"), + &info, + ) + .expect("write partition"); + + let block = UNALIGNED_PART_START * SECTOR_SIZE + block; + let block = block..block + IMAGE_BLOCK_SIZE as u64; + let image_data = + sparse::data_ranges(&image_file, block.start, block.end).expect("data ranges"); + assert_eq!(image_data, vec![block], "zero block is still a hole"); + } + + #[test] + fn write_partition_maps_zero_block_written_into_a_hole() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic"); + let partition = dir.path().join("partition.img"); + let image_file = fs::File::create(&image).expect("create image"); + image_file + .set_len((PART_START + PART_SECTORS) * SECTOR_SIZE) + .expect("set image size"); + let partition_file = fs::File::create(&partition).expect("create partition"); + partition_file + .set_len(PART_SECTORS * SECTOR_SIZE) + .expect("set partition size"); + let block = (ZEROED_BLOCK * IMAGE_BLOCK_SIZE) as u64; + partition_file + .write_all_at(&[0; IMAGE_BLOCK_SIZE], block) + .expect("write zero block"); + + write_partition( + image.to_str().expect("image path"), + partition.to_str().expect("partition path"), + &partition_info(), + ) + .expect("write partition"); + + let block = PART_START * SECTOR_SIZE + block; + assert_eq!( + sparse::data_ranges( + &image_file, + 0, + image_file.metadata().expect("metadata").len() + ) + .expect("data ranges"), + vec![block..block + IMAGE_BLOCK_SIZE as u64] + ); + } +} diff --git a/src/file/mod.rs b/src/file/mod.rs index 6f9db48..92d9c67 100644 --- a/src/file/mod.rs +++ b/src/file/mod.rs @@ -1,6 +1,10 @@ +pub mod bmap; pub mod compression; pub mod functions; +pub(crate) mod output; mod partition; +pub(crate) mod sparse; + use super::validators::{ device_update, identity::{IdentityConfig, IdentityType, validate_identity}, @@ -15,6 +19,14 @@ use std::{ path::{Path, PathBuf}, }; +/// Granularity at which zero data in an image file is kept as a hole. +const IMAGE_BLOCK_SIZE: usize = 4096; +const COPY_CHUNK_SIZE: usize = 1024 * 1024; + +fn is_zero(data: &[u8]) -> bool { + data.iter().all(|b| *b == 0) +} + pub fn set_iotedge_gateway_config( config_file: &Path, image_file: &Path, diff --git a/src/file/output.rs b/src/file/output.rs new file mode 100644 index 0000000..d52dd02 --- /dev/null +++ b/src/file/output.rs @@ -0,0 +1,508 @@ +use crate::file::{bmap, compression::Compression}; +use anyhow::{Context, Result}; +use log::warn; +use std::{ + collections::BTreeSet, + fs::{self, File}, + io::ErrorKind, + iter, + path::{Path, PathBuf}, +}; +use tempfile::TempDir; + +const TMP_DIR_PREFIX: &str = ".omnect-cli-"; + +/// Returns `explicit`, else `image` if it has an extension of `target`, else `image` without the +/// extension of its `source` compression, with the extension of `target` appended. Fails if +/// `explicit` has no extension of `target`, or a compression extension while `target` is none. +pub fn output_image( + image: &Path, + source: Option<&Compression>, + target: Option<&Compression>, + explicit: Option<&Path>, +) -> Result { + if let Some(path) = explicit { + match target { + Some(c) => anyhow::ensure!( + c.matches_extension(path), + "{} has no extension of the -p compression; name it e.g. {}", + path.display(), + c.compressed_path(path).display() + ), + None => anyhow::ensure!( + !Compression::has_extension(path), + "{} has a compression extension, but the image is written uncompressed; pass -p", + path.display() + ), + } + return Ok(path.to_path_buf()); + } + if target.is_some_and(|c| c.matches_extension(image)) { + return Ok(image.to_path_buf()); + } + + let mut output = image.to_path_buf(); + if source.is_some_and(|c| c.matches_extension(image)) { + output.set_extension(""); + } + Ok(match target { + Some(c) => c.compressed_path(&output), + None => output, + }) +} + +/// Fails if writing `output_image` and its bmap would overwrite a file the user did not name, +/// replace the bmap of an image that stays, or leave a bmap that `bmaptool` would use for the +/// output image and that does not match it. +pub fn check_conflicts( + input_image: &Path, + output_image: &Path, + explicit_output: bool, + output_bmap: bool, + input_bmap: Option<&Path>, +) -> Result<()> { + let in_place = same_file(input_image, output_image)?; + + anyhow::ensure!( + explicit_output || in_place || !output_image.exists(), + "{} exists; choose the output image with --output-image", + output_image.display() + ); + if output_bmap { + let target_bmap = bmap::bmap_path(output_image); + if !in_place { + let found_bmap = bmap::find_input_bmap(input_image, None)?; + for bmap in input_bmap.into_iter().chain(found_bmap.as_deref()) { + anyhow::ensure!( + !same_file(bmap, &target_bmap)?, + "{} is the bmap of {} and would be replaced; choose another output image \ + with --output-image", + bmap.display(), + input_image.display() + ); + } + } + } else if let Some(stale_bmap) = bmap::find_input_bmap(output_image, None)? { + anyhow::bail!( + "{} would not match {}; pass -b to write a matching bmap", + stale_bmap.display(), + output_image.display() + ); + } + + Ok(()) +} + +/// A new file in the directory of its destination, which `commit` renames to the destination. +/// Until then the destination stays unchanged. +pub struct Staged { + _tmp_dir: TempDir, + file: PathBuf, + dest: PathBuf, +} + +impl Staged { + /// Writes and syncs the new file for `dest`. A symlink is followed, and an existing `dest` + /// keeps its permissions. + pub fn write(dest: &Path, write: impl FnOnce(&mut File) -> Result<()>) -> Result { + let dest = resolve(dest)?; + let dir = dest + .parent() + .context(format!("{} has no directory", dest.display()))?; + let tmp_dir = tempfile::Builder::new() + .prefix(TMP_DIR_PREFIX) + .tempdir_in(dir) + .context(format!("cannot create a temp dir in {}", dir.display()))?; + let file = tmp_dir.path().join( + dest.file_name() + .context(format!("{} has no file name", dest.display()))?, + ); + + let mut handle = + File::create(&file).context(format!("cannot create {}", file.display()))?; + write(&mut handle)?; + if let Ok(metadata) = fs::metadata(&dest) { + handle + .set_permissions(metadata.permissions()) + .context(format!("cannot set permissions of {}", file.display()))?; + } + // the handle that wrote the data also gets the write errors + handle + .sync_all() + .context(format!("cannot sync {}", file.display()))?; + + Ok(Self { + _tmp_dir: tmp_dir, + file, + dest, + }) + } + + fn rename(&self) -> Result<()> { + fs::rename(&self.file, &self.dest).context(format!( + "cannot rename {} to {}", + self.file.display(), + self.dest.display() + )) + } +} + +/// Renames the staged `image` and its `bmap`, then syncs their directories, so the renames are +/// not lost on a power loss. The old bmap is removed before the image is renamed, so if renaming +/// the bmap fails, no bmap is left that does not match the new image. +pub fn commit(image: Staged, bmap: Option) -> Result<()> { + if let Some(bmap) = &bmap { + match fs::remove_file(&bmap.dest) { + Err(e) if e.kind() != ErrorKind::NotFound => { + return Err(e).context(format!("cannot remove {}", bmap.dest.display())); + } + _ => {} + } + } + image.rename()?; + if let Some(bmap) = &bmap { + bmap.rename()?; + } + + let dirs: BTreeSet<_> = iter::once(&image) + .chain(&bmap) + .filter_map(|staged| staged.dest.parent()) + .collect(); + for dir in dirs { + // both files are complete already, only a power loss could undo the rename + if let Err(e) = File::open(dir).and_then(|dir| dir.sync_all()) { + warn!("cannot sync {}: {e}", dir.display()); + } + } + + Ok(()) +} + +fn same_file(a: &Path, b: &Path) -> Result { + Ok(resolve(a)? == resolve(b)?) +} + +/// The absolute path of `path` with symlinks resolved. A file that does not exist yet is +/// resolved through its directory. +fn resolve(path: &Path) -> Result { + if path.exists() { + return fs::canonicalize(path).context(format!("cannot resolve {}", path.display())); + } + let dir = match path.parent() { + Some(dir) if !dir.as_os_str().is_empty() => dir, + _ => Path::new("."), + }; + let name = path + .file_name() + .context(format!("{} has no file name", path.display()))?; + + Ok(fs::canonicalize(dir) + .context(format!("cannot resolve {}", dir.display()))? + .join(name)) +} + +#[cfg(test)] +mod tests { + use crate::file::output::*; + use std::{ + io::Write, + os::unix::fs::{PermissionsExt, symlink}, + }; + + const IMAGE: &str = "image.wic"; + const IMAGE_XZ: &str = "image.wic.xz"; + const OTHER_IMAGE: &str = "device.wic"; + const XZ_LEVEL: u32 = 9; + const OLD_CONTENT: &[u8] = b"old image"; + const NEW_CONTENT: &[u8] = b"new image"; + const IMAGE_MODE: u32 = 0o644; + const MODE_MASK: u32 = 0o777; + + fn touch(path: &Path) { + fs::write(path, []).expect("create file"); + } + + #[test] + fn output_image_follows_compression() { + let image = Path::new(IMAGE); + let image_xz = Path::new(IMAGE_XZ); + let xz = Compression::xz { + compression_level: XZ_LEVEL, + }; + let output = |image, source, target| { + output_image(image, source, target, None).expect("output image") + }; + + assert_eq!(output(image, None, None), image); + assert_eq!(output(image_xz, Some(&xz), None), image); + assert_eq!(output(image, None, Some(&xz)), image_xz); + assert_eq!(output(image_xz, Some(&xz), Some(&xz)), image_xz); + } + + #[test] + fn output_image_keeps_every_extension_of_the_same_compression() { + for (image, compression) in [ + ("image.wic.gz", Compression::gzip), + ("image.wic.gzip", Compression::gzip), + ("image.wic.bz2", Compression::bzip2), + ("image.wic.bzip2", Compression::bzip2), + ] { + let image = Path::new(image); + assert_eq!( + output_image(image, Some(&compression), Some(&compression), None) + .expect("output image"), + image + ); + } + } + + #[test] + fn output_image_strips_only_the_source_compression_extension() { + let image = Path::new("image.bin"); + let xz = Compression::xz { + compression_level: XZ_LEVEL, + }; + + assert_eq!( + output_image(image, Some(&xz), None, None).expect("output image"), + image + ); + } + + #[test] + fn output_image_takes_explicit_path_with_matching_extension() { + let xz = Compression::xz { + compression_level: XZ_LEVEL, + }; + let other_xz = Path::new("device.wic.xz"); + + assert_eq!( + output_image( + Path::new(IMAGE_XZ), + Some(&xz), + None, + Some(Path::new(OTHER_IMAGE)) + ) + .expect("uncompressed"), + Path::new(OTHER_IMAGE) + ); + assert_eq!( + output_image(Path::new(IMAGE), None, Some(&xz), Some(other_xz)).expect("compressed"), + other_xz + ); + } + + #[test] + fn output_image_rejects_explicit_path_with_wrong_extension() { + let xz = Compression::xz { + compression_level: XZ_LEVEL, + }; + + assert!( + output_image( + Path::new(IMAGE), + None, + Some(&xz), + Some(Path::new(OTHER_IMAGE)) + ) + .is_err() + ); + assert!( + output_image( + Path::new(IMAGE), + None, + Some(&xz), + Some(Path::new("device.wic.gz")) + ) + .is_err() + ); + assert!( + output_image( + Path::new(IMAGE_XZ), + Some(&xz), + None, + Some(Path::new("device.wic.xz")) + ) + .is_err() + ); + } + + #[test] + fn check_conflicts_allows_in_place_with_bmap() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + let image_bmap = bmap::bmap_path(&image); + touch(&image); + touch(&image_bmap); + + check_conflicts(&image, &image, false, true, Some(&image_bmap)).expect("no conflict"); + } + + #[test] + fn check_conflicts_rejects_existing_derived_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image_xz = dir.path().join(IMAGE_XZ); + let image = dir.path().join(IMAGE); + touch(&image_xz); + touch(&image); + + assert!(check_conflicts(&image_xz, &image, false, false, None).is_err()); + check_conflicts(&image_xz, &image, true, false, None).expect("explicit output"); + } + + #[test] + fn check_conflicts_rejects_replacing_the_input_bmap() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image_xz = dir.path().join(IMAGE_XZ); + let image = dir.path().join(IMAGE); + let image_bmap = bmap::bmap_path(&image); + touch(&image_xz); + touch(&image_bmap); + + assert!(check_conflicts(&image_xz, &image, false, true, Some(&image_bmap)).is_err()); + let other = dir.path().join(OTHER_IMAGE); + check_conflicts(&image_xz, &other, true, true, Some(&image_bmap)).expect("other output"); + } + + #[test] + fn check_conflicts_rejects_leaving_a_stale_bmap() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + touch(&image); + touch(&bmap::bmap_path(&image)); + + assert!(check_conflicts(&image, &image, false, false, None).is_err()); + } + + #[test] + fn check_conflicts_rejects_a_stale_bmap_bmaptool_finds() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + touch(&image); + touch(&dir.path().join("image.bmap")); + + assert!(check_conflicts(&image, &image, false, false, None).is_err()); + } + + #[test] + fn check_conflicts_rejects_a_stale_bmap_of_the_uncompressed_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image_xz = dir.path().join(IMAGE_XZ); + let image_bmap = bmap::bmap_path(&dir.path().join(IMAGE)); + touch(&image_xz); + touch(&image_bmap); + + // bmaptool uses image.wic.bmap for image.wic.xz + assert!(check_conflicts(&image_xz, &image_xz, false, false, None).is_err()); + check_conflicts(&image_xz, &image_xz, false, true, Some(&image_bmap)).expect("with -b"); + } + + #[test] + fn check_conflicts_rejects_replacing_the_bmap_next_to_the_input_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image_xz = dir.path().join(IMAGE_XZ); + let image = dir.path().join(IMAGE); + let other_bmap = dir.path().join("other.bmap"); + touch(&image_xz); + touch(&bmap::bmap_path(&image)); + touch(&other_bmap); + + // --input-bmap and --no-input-bmap + assert!(check_conflicts(&image_xz, &image, false, true, Some(&other_bmap)).is_err()); + assert!(check_conflicts(&image_xz, &image, false, true, None).is_err()); + } + + fn stage(dest: &Path, content: &'static [u8]) -> Staged { + Staged::write(dest, |file| Ok(file.write_all(content)?)).expect("stage") + } + + #[test] + fn staged_write_keeps_dest_when_write_fails() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + fs::write(&image, OLD_CONTENT).expect("write image"); + + assert!( + Staged::write(&image, |file| { + file.write_all(NEW_CONTENT).expect("write file"); + anyhow::bail!("write failed") + }) + .is_err() + ); + assert_eq!(fs::read(&image).expect("read image"), OLD_CONTENT); + assert_eq!( + fs::read_dir(dir.path()).expect("read dir").count(), + 1, + "temp dir was left" + ); + } + + #[test] + fn staged_commit_keeps_permissions_and_symlink() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + let link = dir.path().join(OTHER_IMAGE); + fs::write(&image, OLD_CONTENT).expect("write image"); + fs::set_permissions(&image, fs::Permissions::from_mode(IMAGE_MODE)) + .expect("set permissions"); + symlink(&image, &link).expect("create symlink"); + + commit(stage(&link, NEW_CONTENT), None).expect("commit"); + + assert!(link.is_symlink()); + assert_eq!(fs::read(&image).expect("read image"), NEW_CONTENT); + assert_eq!( + fs::metadata(&image).expect("metadata").permissions().mode() & MODE_MASK, + IMAGE_MODE + ); + } + + #[test] + fn commit_leaves_no_old_bmap_when_the_bmap_rename_fails() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + let image_bmap = bmap::bmap_path(&image); + fs::write(&image, OLD_CONTENT).expect("write image"); + fs::write(&image_bmap, OLD_CONTENT).expect("write bmap"); + + let staged_bmap = stage(&image_bmap, NEW_CONTENT); + fs::remove_file(&staged_bmap.file).expect("remove staged bmap"); + + assert!(commit(stage(&image, NEW_CONTENT), Some(staged_bmap)).is_err()); + assert_eq!(fs::read(&image).expect("read image"), NEW_CONTENT); + assert!(!image_bmap.exists()); + } + + #[test] + fn commit_replaces_the_target_of_a_symlinked_bmap() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + let image_bmap = bmap::bmap_path(&image); + let target_bmap = dir.path().join("target.bmap"); + fs::write(&image, OLD_CONTENT).expect("write image"); + fs::write(&target_bmap, OLD_CONTENT).expect("write bmap"); + symlink(&target_bmap, &image_bmap).expect("create symlink"); + + commit( + stage(&image, NEW_CONTENT), + Some(stage(&image_bmap, NEW_CONTENT)), + ) + .expect("commit"); + + assert!(image_bmap.is_symlink()); + assert_eq!(fs::read(&target_bmap).expect("read bmap"), NEW_CONTENT); + } + + #[test] + fn same_file_resolves_relative_and_missing_paths() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + touch(&image); + let relative = dir.path().join(".").join(IMAGE); + let missing = dir.path().join(OTHER_IMAGE); + + assert!(same_file(&image, &relative).expect("same file")); + assert!(same_file(&missing, &missing).expect("same file")); + assert!(!same_file(&image, &missing).expect("same file")); + } +} diff --git a/src/file/partition.rs b/src/file/partition.rs index 04d2fc6..747c8f4 100644 --- a/src/file/partition.rs +++ b/src/file/partition.rs @@ -4,6 +4,8 @@ use std::io::Seek; use std::io::SeekFrom; use std::path::Path; +pub const SECTOR_SIZE: u64 = 512; + pub struct PartitionData { pub num: u32, pub start: u64, @@ -29,10 +31,12 @@ pub fn get_partition_data>(path: P, partition_num: u32) -> Result entry.ending_lba >= entry.starting_lba, "GPT partition {partition_num} has invalid LBA range (ending_lba < starting_lba)" ); + // the callers count in SECTOR_SIZE units + let scale = gpt.sector_size / SECTOR_SIZE; return Ok(PartitionData { num: partition_num, - start: entry.starting_lba, - count: entry.ending_lba - entry.starting_lba + 1, + start: entry.starting_lba * scale, + count: (entry.ending_lba - entry.starting_lba + 1) * scale, }); } Err(e) => e, @@ -41,7 +45,7 @@ pub fn get_partition_data>(path: P, partition_num: u32) -> Result // Try MBR — iter() includes logical partitions (5, 6, ...) file.seek(SeekFrom::Start(0)) .context("failed to seek to start of image")?; - let mbr = mbrman::MBR::read_from(&mut file, 512) + let mbr = mbrman::MBR::read_from(&mut file, SECTOR_SIZE as u32) .with_context(|| format!("image is neither valid GPT nor MBR (GPT error: {gpt_err})"))?; for (i, p) in mbr.iter() { if u32::try_from(i).ok() == Some(partition_num) && p.is_used() { @@ -59,3 +63,40 @@ pub fn is_gpt>(path: P) -> Result { let mut file = File::open(path.as_ref()).context("is_gpt: failed to open image")?; Ok(gptman::GPT::find_from(&mut file).is_ok()) } + +#[cfg(test)] +mod tests { + use crate::file::partition::*; + use gptman::{GPT, GPTPartitionEntry}; + + const GPT_SECTOR_SIZE: u64 = 4096; + const DISK_SECTORS: u64 = 64; + const PART_NUM: u32 = 1; + const PART_SECTORS: u64 = 8; + const GUID: [u8; 16] = [0xff; 16]; + + #[test] + fn get_partition_data_converts_4k_gpt_sectors() { + let dir = tempfile::tempdir().expect("create temp dir"); + let path = dir.path().join("image.wic"); + let mut file = File::create(&path).expect("create image"); + file.set_len(DISK_SECTORS * GPT_SECTOR_SIZE) + .expect("set image size"); + let mut gpt = GPT::new_from(&mut file, GPT_SECTOR_SIZE, GUID).expect("create gpt"); + let start = gpt.header.first_usable_lba; + gpt[PART_NUM] = GPTPartitionEntry { + partition_type_guid: GUID, + unique_partition_guid: GUID, + starting_lba: start, + ending_lba: start + PART_SECTORS - 1, + attribute_bits: 0, + partition_name: "part".into(), + }; + gpt.write_into(&mut file).expect("write gpt"); + + let data = get_partition_data(&path, PART_NUM).expect("partition data"); + let scale = GPT_SECTOR_SIZE / SECTOR_SIZE; + assert_eq!(data.start, start * scale); + assert_eq!(data.count, PART_SECTORS * scale); + } +} diff --git a/src/file/sparse.rs b/src/file/sparse.rs new file mode 100644 index 0000000..7953b8e --- /dev/null +++ b/src/file/sparse.rs @@ -0,0 +1,187 @@ +use anyhow::{Context, Result}; +use rustix::{ + fs::{SeekFrom, seek}, + io::Errno, +}; +use std::{ + fs::File, + io::{self, Read, Seek}, + ops::Range, + path::Path, +}; + +/// Returns the byte ranges of `file` between `start` and `end` that hold data. +pub fn data_ranges(file: &File, start: u64, end: u64) -> Result>> { + let mut ranges = vec![]; + let mut pos = start; + while pos < end { + let data = match seek(file, SeekFrom::Data(pos)) { + Ok(data) => data, + // no data after pos + Err(Errno::NXIO) => break, + Err(e) => return Err(e).context("data_ranges: cannot find data"), + }; + if data >= end { + break; + } + let hole = seek(file, SeekFrom::Hole(data)).context("data_ranges: cannot find hole")?; + ranges.push(data..hole.min(end)); + pos = hole; + } + + Ok(ranges) +} + +/// Sorts `ranges` and joins the ones that overlap or touch. +pub fn merge(mut ranges: Vec>) -> Vec> { + ranges.sort_by_key(|r| r.start); + let mut merged: Vec> = Vec::with_capacity(ranges.len()); + for range in ranges { + match merged.last_mut() { + Some(last) if range.start <= last.end => last.end = last.end.max(range.end), + _ => merged.push(range), + } + } + merged +} + +/// Copies `len` bytes of `src` at `src_offset` to `dst` at `dst_offset`. Holes of `src` are +/// skipped, so they stay holes in a new `dst`. +pub fn copy_range( + src: &File, + src_offset: u64, + dst: &File, + dst_offset: u64, + len: u64, +) -> Result<()> { + let src_len = src + .metadata() + .context("copy_range: cannot get source size")? + .len(); + anyhow::ensure!( + src_offset + len <= src_len, + "copy_range: source has {src_len} bytes, cannot copy {len} bytes at {src_offset}" + ); + + for data in data_ranges(src, src_offset, src_offset + len)? { + let mut src = src; + let mut dst = dst; + src.seek(io::SeekFrom::Start(data.start)) + .context("copy_range: cannot seek source")?; + dst.seek(io::SeekFrom::Start(data.start - src_offset + dst_offset)) + .context("copy_range: cannot seek destination")?; + // one copy_file_range call copies at most 2 GiB; io::copy repeats it until a range is done + let copied = io::copy(&mut src.take(data.end - data.start), &mut dst) + .context("copy_range: cannot copy data")?; + anyhow::ensure!( + copied == data.end - data.start, + "copy_range: source ended early" + ); + } + + Ok(()) +} + +/// Creates `dst` with `len` bytes and copies `len` bytes of `src` at `src_offset` into it. +/// Holes of `src` stay holes in `dst`. +pub fn copy_to_new_file(src: &File, src_offset: u64, dst: &Path, len: u64) -> Result<()> { + let dst = + File::create(dst).context(format!("copy_to_new_file: cannot create {}", dst.display()))?; + + copy_to_empty_file(src, src_offset, &dst, len) +} + +/// Sets the size of the empty `dst` to `len` and copies `len` bytes of `src` at `src_offset` +/// into it. Holes of `src` stay holes in `dst`. +pub fn copy_to_empty_file(src: &File, src_offset: u64, dst: &File, len: u64) -> Result<()> { + dst.set_len(len) + .context("copy_to_empty_file: cannot set destination size")?; + + copy_range(src, src_offset, dst, 0, len) +} + +#[cfg(test)] +mod tests { + use crate::file::sparse::*; + use std::os::unix::fs::FileExt; + + const BLOCK_SIZE: u64 = 4096; + const FILE_BLOCKS: u64 = 8; + const DATA_BLOCK: u64 = 2; + const DATA_BYTE: u8 = 0x5a; + const DST_OFFSET_BLOCKS: u64 = 1; + + /// A sparse file with data in `DATA_BLOCK` and holes everywhere else. + fn sparse_file(dir: &std::path::Path) -> File { + let file = File::options() + .read(true) + .write(true) + .create_new(true) + .open(dir.join("src")) + .expect("create file"); + file.set_len(FILE_BLOCKS * BLOCK_SIZE).expect("set size"); + file.write_all_at(&[DATA_BYTE; BLOCK_SIZE as usize], DATA_BLOCK * BLOCK_SIZE) + .expect("write data block"); + file + } + + #[test] + fn data_ranges_finds_data_between_holes() { + let dir = tempfile::tempdir().expect("create temp dir"); + let file = sparse_file(dir.path()); + let data = DATA_BLOCK * BLOCK_SIZE..(DATA_BLOCK + 1) * BLOCK_SIZE; + + assert_eq!( + data_ranges(&file, 0, FILE_BLOCKS * BLOCK_SIZE).expect("data ranges"), + vec![data.clone()] + ); + assert_eq!( + data_ranges(&file, 0, data.start).expect("data ranges"), + vec![] + ); + assert_eq!( + data_ranges(&file, data.start + 1, data.end - 1).expect("data ranges"), + vec![data.start + 1..data.end - 1] + ); + } + + #[test] + fn merge_joins_overlapping_and_touching_ranges() { + assert_eq!( + merge(vec![8..10, 0..2, 1..4, 4..5, 12..12]), + vec![0..5, 8..10, 12..12] + ); + } + + #[test] + fn copy_to_new_file_keeps_holes() { + let dir = tempfile::tempdir().expect("create temp dir"); + let src = sparse_file(dir.path()); + let dst_path = dir.path().join("dst"); + let len = (FILE_BLOCKS - DST_OFFSET_BLOCKS) * BLOCK_SIZE; + + copy_to_new_file(&src, DST_OFFSET_BLOCKS * BLOCK_SIZE, &dst_path, len) + .expect("copy to new file"); + + let dst = File::open(&dst_path).expect("open file"); + assert_eq!(dst.metadata().expect("metadata").len(), len); + let data_start = (DATA_BLOCK - DST_OFFSET_BLOCKS) * BLOCK_SIZE; + assert_eq!( + data_ranges(&dst, 0, len).expect("data ranges"), + vec![data_start..data_start + BLOCK_SIZE] + ); + let mut block = [0u8; BLOCK_SIZE as usize]; + dst.read_exact_at(&mut block, data_start) + .expect("read block"); + assert!(block.iter().all(|b| *b == DATA_BYTE)); + } + + #[test] + fn copy_range_rejects_range_after_source_end() { + let dir = tempfile::tempdir().expect("create temp dir"); + let src = sparse_file(dir.path()); + let dst = File::create(dir.path().join("dst")).expect("create file"); + + assert!(copy_range(&src, BLOCK_SIZE, &dst, 0, FILE_BLOCKS * BLOCK_SIZE).is_err()); + } +} diff --git a/src/lib.rs b/src/lib.rs index 343f912..95ba6e5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,139 +17,184 @@ use cli::{ SetConfig, SetDeviceCertificate, SetDeviceCertificateNoEst, SetEdgeCaCertificate, SetIotLeafSasConfig, SetIotedgeGatewayConfig, }, + ImageOutput, IotHubDeviceUpdate::{self, SetDeviceConfig as IotHubDeviceUpdateSet}, SshConfig::{SetCertificate, SetConnection}, }; -use file::{compression::Compression, functions::FileCopyToParams}; -use log::error; -use std::{fs, path::Path, path::PathBuf}; -use tokio::fs::remove_dir_all; -use uuid::Uuid; - -use crate::file::compression; - -struct TempDirGuard(PathBuf); - -impl Drop for TempDirGuard { - fn drop(&mut self) { - let Ok(rt) = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - else { - error!("cannot create tokio runtime"); - return; - }; - - rt.block_on(async { - if let Err(e) = remove_dir_all(self.0.clone()).await { - error!("cannot remove tmp dir: {e}") - } - }) - } -} +use file::{ + bmap::{self, Bmap}, + compression::{self, Compression}, + functions::FileCopyToParams, + output::{self, Staged, check_conflicts, output_image}, + sparse, +}; +use log::info; +use std::{ + fs, io, + path::{Path, PathBuf}, +}; -fn run_image_command( - image_file: PathBuf, - generate_bmap: bool, - target_compression: Option, - command: F, -) -> Result<()> +fn run_image_command(image_file: PathBuf, output: ImageOutput, command: F) -> Result<()> where F: FnOnce(&PathBuf) -> Result<()>, { - if let Ok("true") | Ok("1") = std::env::var("CONTAINERIZED").as_deref() { - anyhow::ensure!( - !generate_bmap, - "run_image_command: generating bmap file is not supported in containerized environments." - ); - } - - anyhow::ensure!( - image_file.try_exists().is_ok_and(|exists| exists), - "run_image_command: image doesn't exist {}", - image_file.to_str().context("cannot get image file path")? - ); - - let mut dest_image_file = image_file.clone(); - - // create /tmp/{uuid}/ and copy image into - let tmp_dir = PathBuf::from(format!("/tmp/{}", Uuid::new_v4())); - fs::create_dir_all(tmp_dir.clone()).context(format!( - "run_image_command: couldn't create destination path {}", - tmp_dir.to_str().context("cannot get tmp dir name")? - ))?; - - let _guard = TempDirGuard(tmp_dir.clone()); + let ImageOutput { + output_bmap, + input_bmap, + no_input_bmap, + output_image: explicit_output, + compress_image: target_compression, + } = output; + + ensure_image_exists(&image_file)?; + + let input_bmap = if output_bmap && !no_input_bmap { + bmap::find_input_bmap(&image_file, input_bmap.as_deref())? + .map(|path| { + Bmap::read(&path) + .with_context(|| input_bmap_error(&path)) + .map(|bmap| (path, bmap)) + }) + .transpose()? + } else { + None + }; + let source_compression = Compression::from_file(&image_file)?; + let dest_image_file = output_image( + &image_file, + source_compression.as_ref(), + target_compression.as_ref(), + explicit_output.as_deref(), + )?; + check_conflicts( + &image_file, + &dest_image_file, + explicit_output.is_some(), + output_bmap, + input_bmap.as_ref().map(|(path, _)| path.as_path()), + )?; - let mut tmp_image_file = tmp_dir.join( - image_file - .file_name() - .context("cannot get image file name")?, - ); + let verify_input_bmap = |image: &Path| -> Result<()> { + let Some((path, bmap)) = &input_bmap else { + return Ok(()); + }; + bmap::verify_checksums(image, bmap).with_context(|| input_bmap_error(path)) + }; - // if applicable decompress image to *.wic - if let Some(source_compression) = Compression::from_file(&image_file)? { - std::fs::copy(&image_file, &tmp_image_file)?; - tmp_image_file = compression::decompress(&tmp_image_file, &source_compression)?; - dest_image_file.set_extension(""); - } else { - // copy sparse file (std::fs::copy isn't able) - libfs::copy_file(&image_file, &tmp_image_file).context(format!( - "error: libfs::copy_file({:?}, {:?})", - image_file, tmp_image_file - ))?; + // a wrong bmap of an uncompressed image fails before the copy + if source_compression.is_none() { + verify_input_bmap(&image_file)?; + } + let tmp_dir = tempfile::tempdir().context("cannot create a temp dir")?; + let tmp_image_file = temp_image(&image_file, source_compression.as_ref(), tmp_dir.path())?; + if source_compression.is_some() { + verify_input_bmap(&tmp_image_file)?; } // run command command(&tmp_image_file)?; - // create and copy back bmap file if one was created - if generate_bmap { - let mut target_bmap = image_file - .parent() - .context("cannot get parent dir of image path")? - .to_path_buf(); - let tmp_bmap = PathBuf::from(format!( - "{}.bmap", - tmp_image_file - .to_str() - .context("cannot get image file path")? - )); - file::functions::generate_bmap_file( - tmp_image_file - .to_str() - .context("cannot get image file path")?, + let tmp_bmap = bmap::bmap_path(&tmp_image_file); + if output_bmap { + match &input_bmap { + Some((path, _)) => info!("the bmap maps the ranges of {}", path.display()), + None => info!("no input bmap, so the bmap maps the whole image"), + } + bmap::write_bmap( + &tmp_image_file, + input_bmap.as_ref().map(|(_, bmap)| bmap), + &tmp_bmap, )?; - target_bmap.push(tmp_bmap.file_name().context("cannot get bmap file name")?); - std::fs::copy(&tmp_bmap, &target_bmap).context(format!( - "error: std::fs::copy({:?}, {:?})", - tmp_bmap, target_bmap - ))?; } - // if applicable compress image - if let Some(c) = target_compression { - tmp_image_file = compression::compress(&tmp_image_file, &c)?; - dest_image_file.set_file_name( - tmp_image_file - .file_name() - .context("cannot get image file name")?, - ); - std::fs::copy(&tmp_image_file, &dest_image_file).context(format!( - "error: std::fs::copy({:?}, {:?})", - tmp_image_file, dest_image_file - ))?; - } else { - // copy sparse file (std::fs::copy isn't able) - libfs::copy_file(&tmp_image_file, &dest_image_file).context(format!( - "error: libfs::copy_file({:?}, {:?})", - tmp_image_file, dest_image_file - ))?; + let image = Staged::write(&dest_image_file, |dest| match &target_compression { + Some(c) => compression::compress(&tmp_image_file, c, dest), + None => copy_image(&tmp_image_file, dest), + })?; + let bmap = output_bmap + .then(|| { + Staged::write(&bmap::bmap_path(&dest_image_file), |dest| { + let mut src = fs::File::open(&tmp_bmap) + .context(format!("cannot open {}", tmp_bmap.display()))?; + io::copy(&mut src, dest) + .map(drop) + .context(format!("cannot copy {}", tmp_bmap.display())) + }) + }) + .transpose()?; + + output::commit(image, bmap) +} + +fn input_bmap_error(path: &Path) -> String { + format!( + "cannot use input bmap {}; pass the right one with --input-bmap, or map the whole \ + image with --no-input-bmap", + path.display() + ) +} + +/// Runs `command` on the image, or on a decompressed temp copy of a compressed image. +fn run_read_only_image_command(image_file: PathBuf, command: F) -> Result<()> +where + F: FnOnce(&PathBuf) -> Result<()>, +{ + ensure_image_exists(&image_file)?; + + match Compression::from_file(&image_file)? { + Some(c) => { + let tmp_dir = tempfile::tempdir().context("cannot create a temp dir")?; + command(&compression::decompress(&image_file, &c, tmp_dir.path())?) + } + None => command(&image_file), } +} +fn ensure_image_exists(image_file: &Path) -> Result<()> { + anyhow::ensure!( + image_file.try_exists().is_ok_and(|exists| exists), + "image doesn't exist {}", + image_file.display() + ); Ok(()) } +/// Copies `image_file` into `tmp_dir`, decompressed if it is compressed, and returns the copy. +fn temp_image( + image_file: &Path, + source_compression: Option<&Compression>, + tmp_dir: &Path, +) -> Result { + match source_compression { + Some(c) => compression::decompress(image_file, c, tmp_dir), + None => { + let tmp_image_file = tmp_dir.join( + image_file + .file_name() + .context("cannot get image file name")?, + ); + let tmp_image = fs::File::create(&tmp_image_file) + .context(format!("cannot create {}", tmp_image_file.display()))?; + copy_image(image_file, &tmp_image).context(format!( + "error: copy_image({:?}, {:?})", + image_file, tmp_image_file + ))?; + Ok(tmp_image_file) + } + } +} + +/// Copies `from` to the empty `to` and keeps the holes of a sparse `from`. +fn copy_image(from: &Path, to: &fs::File) -> Result<()> { + let src = fs::File::open(from).context("copy_image: cannot open source")?; + let len = src + .metadata() + .context("copy_image: cannot get source size")? + .len(); + + sparse::copy_to_empty_file(&src, 0, to, len) +} + struct CertInfo { cert_path: PathBuf, key_path: PathBuf, @@ -197,9 +242,8 @@ pub fn run() -> Result<()> { image, partition, dest, - generate_bmap, - compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img| { + output, + }) => run_image_command(image, output, |img| { anyhow::ensure!( dest.to_string_lossy().ends_with(".tar.gz"), format!( @@ -237,9 +281,8 @@ pub fn run() -> Result<()> { config, image, payload, - generate_bmap, - compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img| { + output, + }) => run_image_command(image, output, |img| { file::set_identity_config(&config, img, payload.as_deref()) })?, Command::Identity(SetDeviceCertificate { @@ -248,8 +291,7 @@ pub fn run() -> Result<()> { image, device_id, days, - generate_bmap, - compress_image, + output, }) => { let cert_info = create_image_cert( &image, @@ -264,7 +306,7 @@ pub fn run() -> Result<()> { ) .context("set_device_certificate: could not create certificate")?; - run_image_command(image, generate_bmap, compress_image, |img| { + run_image_command(image, output, |img| { file::set_device_cert( Some(&intermediate_full_chain_cert), &cert_info.cert_path, @@ -279,8 +321,7 @@ pub fn run() -> Result<()> { image, device_id, days, - generate_bmap, - compress_image, + output, }) => { let cert_info = create_image_cert( &image, @@ -295,7 +336,7 @@ pub fn run() -> Result<()> { ) .context("set_edge_ca_certificate: could not create certificate")?; - run_image_command(image, generate_bmap, compress_image, |img| { + run_image_command(image, output, |img| { file::set_edge_ca_cert( Some(&intermediate_full_chain_cert), &cert_info.cert_path, @@ -308,9 +349,8 @@ pub fn run() -> Result<()> { device_cert: device_cert_pem, device_key: device_key_pem, image, - generate_bmap, - compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img| { + output, + }) => run_image_command(image, output, |img| { file::set_device_cert(None, &device_cert_pem, &device_key_pem, img) })?, Command::Identity(SetIotedgeGatewayConfig { @@ -319,9 +359,8 @@ pub fn run() -> Result<()> { root_ca, device_identity, device_identity_key, - generate_bmap, - compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img: &PathBuf| { + output, + }) => run_image_command(image, output, |img: &PathBuf| { file::set_iotedge_gateway_config( &config, img, @@ -334,25 +373,22 @@ pub fn run() -> Result<()> { config, image, root_ca, - generate_bmap, - compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img: &PathBuf| { + output, + }) => run_image_command(image, output, |img: &PathBuf| { file::set_iot_leaf_sas_config(&config, img, &root_ca) })?, Command::Ssh(SetCertificate { image, root_ca, - generate_bmap, - compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img: &PathBuf| { + output, + }) => run_image_command(image, output, |img: &PathBuf| { file::set_ssh_tunnel_certificate(img, &root_ca) })?, Command::IotHubDeviceUpdate(IotHubDeviceUpdateSet { iot_hub_device_update_config, image, - generate_bmap, - compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img: &PathBuf| { + output, + }) => run_image_command(image, output, |img: &PathBuf| { file::set_iot_hub_device_update_config(&iot_hub_device_update_config, img) })?, Command::IotHubDeviceUpdate(IotHubDeviceUpdate::ImportUpdate { @@ -467,15 +503,14 @@ pub fn run() -> Result<()> { Command::File(CopyToImage { file_copy_params, image, - generate_bmap, - compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img: &PathBuf| { + output, + }) => run_image_command(image, output, |img: &PathBuf| { file::copy_to_image(&file_copy_params, img) })?, Command::File(CopyFromImage { file_copy_params, image, - }) => run_image_command(image, false, None, |img: &PathBuf| { + }) => run_read_only_image_command(image, |img: &PathBuf| { file::copy_from_image(&file_copy_params, img) })?, } diff --git a/testfiles/image.wic.bmap b/testfiles/image.wic.bmap new file mode 100644 index 0000000..7b84de9 --- /dev/null +++ b/testfiles/image.wic.bmap @@ -0,0 +1,69 @@ + + + + + + 30408704 + + + 4096 + + + 7424 + + + 77 + + + sha256 + + + 43b997b9550b07ad422eddddba268d0f0607660f664592a7d749034a4f8b9eea + + + + 0-1 + 1024-1025 + 2048-2055 + 2058-2059 + 2062-2063 + 2190-2191 + 3072-3079 + 3082-3083 + 4094-4103 + 4106-4107 + 5104-5105 + 5120-5127 + 5130-5131 + 6128-6129 + 6144-6151 + 6154-6155 + 7152-7153 + 7168-7175 + 7178-7179 + 7423 + + diff --git a/testfiles/image.wic.xz b/testfiles/image.wic.xz index b86e932..6053d9a 100644 Binary files a/testfiles/image.wic.xz and b/testfiles/image.wic.xz differ diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index ac3e716..d0b358e 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -3,9 +3,14 @@ use assert_cmd::Command; use assert_json_diff::assert_json_eq; use common::Testrunner; use httpmock::prelude::*; -use omnect_cli::ssh; +use omnect_cli::{file::bmap::Bmap, ssh}; use predicates::prelude::*; -use std::{fs::create_dir_all, path::PathBuf}; +use std::{ + ffi::OsStr, + fs::{self, create_dir_all}, + os::unix::fs::MetadataExt, + path::{Path, PathBuf}, +}; use stdext::function_name; #[test] @@ -895,6 +900,472 @@ fn check_file_copy(tr: Testrunner, partition: &str) { assert!(file_diff::diff(in_file4, out_file4)); } +// unit of `st_blocks` +const STAT_BLOCK_SIZE: u64 = 512; +// content an earlier installation left on the device +const OLD_CONTENT_BYTE: u8 = 0xa5; +// a file with a zero range that spans whole image blocks +const ZERO_BLOCKS_FILE_EDGE: usize = 4096; +const ZERO_BLOCKS_FILE_ZEROS: usize = 64 * 1024; +const OTHER_IMAGE_BMAP: &str = "40964096\ + 0"; + +fn bmap_path_of(image: &Path) -> PathBuf { + PathBuf::from(format!("{}.bmap", image.to_string_lossy())) +} + +fn read_bmap(path: &Path) -> Bmap { + Bmap::read(path).expect("read bmap") +} + +fn mapped_blocks(bmap: &Bmap) -> u64 { + bmap.ranges.iter().map(|r| r.end() - r.start() + 1).sum() +} + +fn maps_whole_image(bmap: &Bmap) -> bool { + mapped_blocks(bmap) == bmap.image_size.div_ceil(bmap.block_size) +} + +fn maps_all_ranges_of(bmap: &Bmap, input: &Bmap) -> bool { + input + .ranges + .iter() + .flat_map(|r| r.clone()) + .all(|b| bmap.ranges.iter().any(|r| r.contains(&b))) +} + +/// Writes the blocks `bmap` maps onto a device that still holds old content, as a bmap flash +/// does. +fn bmap_flash(image: &[u8], bmap: &Bmap) -> Vec { + let mut device = vec![OLD_CONTENT_BYTE; image.len()]; + for range in &bmap.ranges { + let start = usize::try_from(range.start() * bmap.block_size).expect("start"); + let end = usize::try_from((range.end() + 1) * bmap.block_size).expect("end"); + let end = end.min(image.len()); + device[start..end].copy_from_slice(&image[start..end]); + } + device +} + +fn sparse_copy(from: &Path, to: &Path) { + let status = std::process::Command::new("cp") + .arg("--sparse=always") + .arg(from) + .arg(to) + .status() + .expect("run cp"); + assert!(status.success(), "sparse copy of the test image failed"); +} + +#[test] +fn check_file_copy_without_input_bmap_maps_whole_image() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let bmap_path = PathBuf::from(format!("{}.bmap", image_path.to_string_lossy())); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},factory:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path) + .arg("-b") + .assert() + .success(); + + assert!(maps_whole_image(&read_bmap(&bmap_path))); +} + +#[test] +fn check_file_copy_keeps_holes() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let input_bmap_path = tr.to_pathbuf("testfiles/image.wic.bmap"); + let sparse_path = image_path.with_file_name("sparse.wic"); + let bmap_path = PathBuf::from(format!("{}.bmap", sparse_path.to_string_lossy())); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + + sparse_copy(&image_path, &sparse_path); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},factory:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&sparse_path) + .arg("--input-bmap") + .arg(&input_bmap_path) + .arg("-b") + .assert() + .success(); + + let meta = sparse_path.metadata().expect("image metadata"); + assert!(meta.blocks() * STAT_BLOCK_SIZE < meta.len()); + let bmap = read_bmap(&bmap_path); + let input_bmap = read_bmap(&input_bmap_path); + assert!(!maps_whole_image(&bmap)); + assert!(maps_all_ranges_of(&bmap, &input_bmap)); + + // every data block and every block of the input bmap must replace the old content + let image = fs::read(&sparse_path).expect("read image"); + let device = bmap_flash(&image, &bmap); + let block_size = usize::try_from(input_bmap.block_size).expect("block size"); + let blocks = image.chunks(block_size).zip(device.chunks(block_size)); + for (block, (image_block, device_block)) in (0u64..).zip(blocks) { + let needed = image_block.iter().any(|b| *b != 0) + || input_bmap.ranges.iter().any(|r| r.contains(&block)); + assert!( + !needed || image_block == device_block, + "block {block} keeps old content" + ); + } +} + +#[test] +fn check_file_copy_maps_zero_blocks_of_copied_files() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let input_bmap_path = tr.to_pathbuf("testfiles/image.wic.bmap"); + let sparse_path = image_path.with_file_name("sparse.wic"); + let bmap_path = PathBuf::from(format!("{}.bmap", sparse_path.to_string_lossy())); + let device_path = tr.pathbuf().join("device.wic"); + let in_file = tr.pathbuf().join("zero-blocks.bin"); + let in_content = [ + vec![0x11; ZERO_BLOCKS_FILE_EDGE], + vec![0; ZERO_BLOCKS_FILE_ZEROS], + vec![0x22; ZERO_BLOCKS_FILE_EDGE], + ] + .concat(); + fs::write(&in_file, &in_content).expect("write input file"); + sparse_copy(&image_path, &sparse_path); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},factory:/my-file", in_file.to_string_lossy())) + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&sparse_path) + .arg("--input-bmap") + .arg(&input_bmap_path) + .arg("-b") + .assert() + .success(); + + let image = fs::read(&sparse_path).expect("read image"); + fs::write(&device_path, bmap_flash(&image, &read_bmap(&bmap_path))).expect("write device"); + + for partition in ["factory", "boot"] { + let out_file = tr.pathbuf().join(format!("{partition}-out.bin")); + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-from-image") + .arg("-f") + .arg(format!( + "{partition}:/my-file,{}", + out_file.to_string_lossy() + )) + .arg("-i") + .arg(&device_path) + .assert() + .success(); + assert!( + fs::read(&out_file).expect("read output file") == in_content, + "{partition}: file differs after a bmap flash" + ); + } +} + +#[test] +fn check_input_bmap_of_another_image() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let bmap_path = PathBuf::from(format!("{}.bmap", image_path.to_string_lossy())); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + let out_file = tr.pathbuf().join("hosts"); + fs::write(&bmap_path, OTHER_IMAGE_BMAP).expect("write bmap"); + let image_hash = Testrunner::file_hash(&image_path); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path) + .arg("-b") + .assert() + .failure() + .stderr(predicate::str::contains("cannot use input bmap")); + assert_eq!(Testrunner::file_hash(&image_path), image_hash); + + // reading from the image does not use the bmap + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-from-image") + .arg("-f") + .arg(format!("rootA:/etc/hosts,{}", out_file.to_string_lossy())) + .arg("-i") + .arg(&image_path) + .assert() + .success(); +} + +#[test] +fn check_file_copy_without_output_bmap_keeps_image_sparse() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let sparse_path = image_path.with_file_name("sparse.wic"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + sparse_copy(&image_path, &sparse_path); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},factory:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&sparse_path) + .assert() + .success(); + + let meta = sparse_path.metadata().expect("image metadata"); + assert!( + meta.blocks() * STAT_BLOCK_SIZE < meta.len(), + "image is not sparse" + ); +} + +#[test] +fn check_no_output_bmap_when_image_write_back_fails() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic.xz"); + let written_image_path = image_path.with_extension(""); + let bmap_path = image_path.with_extension("bmap"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + // a directory as the output image makes the write back fail, also for root + create_dir_all(&written_image_path).expect("create directory"); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path) + .arg("-b") + .arg("-o") + .arg(&written_image_path) + .assert() + .failure() + .stderr(predicate::str::contains("cannot rename")); + assert!( + !bmap_path.exists(), + "bmap written for an image that was not written back" + ); +} + +#[test] +fn check_input_bmap_requires_output_bmap() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let bmap_path = tr.to_pathbuf("testfiles/image.wic.bmap"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + let image_hash = Testrunner::file_hash(&image_path); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path) + .arg("--input-bmap") + .arg(&bmap_path) + .assert() + .failure(); + assert_eq!(Testrunner::file_hash(&image_path), image_hash); +} + +#[test] +fn check_input_bmap_of_another_image_with_the_same_size() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let bmap_path = tr.to_pathbuf("testfiles/image.wic.bmap"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + let bmap = read_bmap(&bmap_path); + // another build: one byte differs in a block the bmap maps + let mapped_byte = usize::try_from(bmap.ranges[0].start() * bmap.block_size).expect("offset"); + let mut image = fs::read(&image_path).expect("read image"); + image[mapped_byte] = !image[mapped_byte]; + fs::write(&image_path, image).expect("write image"); + let image_hash = Testrunner::file_hash(&image_path); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path) + .arg("-b") + .assert() + .failure() + .stderr(predicate::str::contains("cannot use input bmap")) + .stderr(predicate::str::contains("--input-bmap")); + assert_eq!(Testrunner::file_hash(&image_path), image_hash); +} + +#[test] +fn check_chained_commands_with_input_bmap() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_xz_path = tr.to_pathbuf("testfiles/image.wic.xz"); + let image_xz_hash = Testrunner::file_hash(&image_xz_path); + let input_bmap_path = tr.to_pathbuf("testfiles/image.wic.bmap"); + let input_bmap_hash = Testrunner::file_hash(&input_bmap_path); + let device_path = tr.pathbuf().join("device.wic"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + let copy_to_image = |image: &Path, args: &[&OsStr]| { + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(image) + .args(args) + .assert() + }; + + // image.wic would be written next to image.wic.bmap without updating it + copy_to_image(&image_xz_path, &[]) + .failure() + .stderr(predicate::str::contains("pass -b to write a matching bmap")); + // bmaptool would use image.wic.bmap for the changed image.wic.xz + copy_to_image(&image_xz_path, &["-p".as_ref(), "xz".as_ref()]) + .failure() + .stderr(predicate::str::contains("pass -b to write a matching bmap")); + // the bmap of image.wic would replace the bmap of image.wic.xz + copy_to_image(&image_xz_path, &["-b".as_ref()]) + .failure() + .stderr(predicate::str::contains("--output-image")); + copy_to_image(&image_xz_path, &["-b".as_ref(), "--no-input-bmap".as_ref()]) + .failure() + .stderr(predicate::str::contains("--output-image")); + + copy_to_image( + &image_xz_path, + &["-b".as_ref(), "-o".as_ref(), device_path.as_os_str()], + ) + .success(); + copy_to_image(&device_path, &["-b".as_ref()]).success(); + copy_to_image(&device_path, &[]) + .failure() + .stderr(predicate::str::contains("pass -b to write a matching bmap")); + + assert_eq!(Testrunner::file_hash(&image_xz_path), image_xz_hash); + assert_eq!(Testrunner::file_hash(&input_bmap_path), input_bmap_hash); + assert!(maps_all_ranges_of( + &read_bmap(&bmap_path_of(&device_path)), + &read_bmap(&input_bmap_path) + )); +} + +#[test] +fn check_derived_output_image_exists() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_xz_path = tr.to_pathbuf("testfiles/image.wic.xz"); + let image_path = image_xz_path.with_extension(""); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + fs::write(&image_path, "an older image").expect("write image"); + let copy_to_image = |args: &[&OsStr]| { + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_xz_path) + .args(args) + .assert() + }; + + copy_to_image(&[]) + .failure() + .stderr(predicate::str::contains("--output-image")); + copy_to_image(&[ + "-o".as_ref(), + image_path.as_os_str(), + "-p".as_ref(), + "xz".as_ref(), + ]) + .failure() + .stderr(predicate::str::contains( + "has no extension of the -p compression", + )); + // a path given with -o is overwritten + copy_to_image(&["-o".as_ref(), image_path.as_os_str()]).success(); + assert_ne!( + fs::read(&image_path).expect("read image"), + b"an older image" + ); +} + +#[test] +fn check_no_input_bmap() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let bmap_path = bmap_path_of(&image_path); + let other_bmap_path = tr.pathbuf().join("other.bmap"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + fs::write(&bmap_path, OTHER_IMAGE_BMAP).expect("write bmap"); + fs::write(&other_bmap_path, OTHER_IMAGE_BMAP).expect("write bmap"); + let copy_to_image = |args: &[&OsStr]| { + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path) + .args(args) + .assert() + }; + + copy_to_image(&["--no-input-bmap".as_ref()]).failure(); + copy_to_image(&[ + "-b".as_ref(), + "--no-input-bmap".as_ref(), + "--input-bmap".as_ref(), + other_bmap_path.as_os_str(), + ]) + .failure(); + copy_to_image(&["-b".as_ref()]) + .failure() + .stderr(predicate::str::contains("--no-input-bmap")); + + copy_to_image(&["-b".as_ref(), "--no-input-bmap".as_ref()]).success(); + assert!(maps_whole_image(&read_bmap(&bmap_path))); +} + #[test] fn check_bmap_generation_wic() { let tr = Testrunner::new(function_name!().split("::").last().unwrap()); @@ -947,9 +1418,10 @@ fn check_bmap_generation_wic() { fn check_bmap_generation_wic_xz() { let tr = Testrunner::new(function_name!().split("::").last().unwrap()); let image_path_wic_xz = tr.to_pathbuf("testfiles/image.wic.xz"); - let image_path_wic = image_path_wic_xz.with_extension(""); + let image_path_wic = tr.pathbuf().join("device.wic"); let image_path_wic_copy = image_path_wic_xz.with_extension("copy"); - let image_path_bmap = image_path_wic_xz.with_extension("bmap"); + let image_path_bmap = bmap_path_of(&image_path_wic); + let input_bmap = read_bmap(&tr.to_pathbuf("testfiles/image.wic.bmap")); let in_file = tr.to_pathbuf("testfiles/boot.scr"); let in_file = in_file.to_str().unwrap(); @@ -962,9 +1434,15 @@ fn check_bmap_generation_wic_xz() { .arg("-i") .arg(&image_path_wic_xz) .arg("-b") + .arg("-o") + .arg(&image_path_wic) .assert(); assert.success(); + let bmap = read_bmap(&image_path_bmap); + assert!(!maps_whole_image(&bmap)); + assert!(maps_all_ranges_of(&bmap, &input_bmap)); + // use bmaptool to verify that the checksum of the bmap file and the image // still match after the copy operations let assert = Command::new("bmaptool") @@ -976,6 +1454,66 @@ fn check_bmap_generation_wic_xz() { assert.success(); } +#[test] +fn check_bmap_generation_wic_xz_without_input_bmap() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path_wic_xz = tr.to_pathbuf("testfiles/image.wic.xz"); + let image_path_bmap = image_path_wic_xz.with_extension("bmap"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path_wic_xz) + .arg("-b") + .assert() + .success(); + + assert!(maps_whole_image(&read_bmap(&image_path_bmap))); +} + +#[test] +fn check_bmap_of_compressed_output_keeps_input_bmap() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let input_bmap_path = tr.to_pathbuf("testfiles/image.wic.bmap"); + let input_bmap_hash = Testrunner::file_hash(&input_bmap_path); + let image_path_xz = image_path.with_extension("wic.xz"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path) + .arg("-b") + .arg("-p") + .arg("xz") + .assert() + .success(); + + assert_eq!( + Testrunner::file_hash(&input_bmap_path), + input_bmap_hash, + "bmap of the unchanged input image was replaced" + ); + // bmaptool finds .bmap without --bmap and checks the image against it + Command::new("bmaptool") + .arg("copy") + .arg(&image_path_xz) + .arg(image_path.with_extension("copy")) + .assert() + .success() + .stderr(predicate::str::contains("image.wic.xz.bmap")); +} + #[test] fn check_image_compression() { let tr = Testrunner::new(function_name!().split("::").last().unwrap()); @@ -1057,6 +1595,78 @@ fn check_image_decompression() { assert_eq!(image_path_wic_xz_hash1, image_path_wic_xz_hash2); } +#[test] +fn check_copy_from_image_writes_no_image() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path_wic_xz = tr.to_pathbuf("testfiles/image.wic.xz"); + let image_hash = Testrunner::file_hash(&image_path_wic_xz); + let out_file = tr.pathbuf().join("hosts"); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-from-image") + .arg("-f") + .arg(format!("rootA:/etc/hosts,{}", out_file.to_string_lossy())) + .arg("-i") + .arg(&image_path_wic_xz) + .assert() + .success(); + + assert!(out_file.exists()); + assert!(!image_path_wic_xz.with_extension("").exists()); + assert_eq!(Testrunner::file_hash(&image_path_wic_xz), image_hash); +} + +#[test] +fn check_partition_files_stay_out_of_the_image_dir() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + // the name of the file that holds partition 1 + let image_path = tr.pathbuf().join("1.img"); + fs::rename(tr.to_pathbuf("testfiles/image.wic"), &image_path).expect("rename image"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + let out_file = tr.pathbuf().join("my-file"); + let file_image = |command: &str, files: String| { + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg(command) + .arg("-f") + .arg(files) + .arg("-i") + .arg(&image_path) + .assert() + .success(); + }; + let dir_entries = || { + let mut entries = fs::read_dir(tr.pathbuf()) + .expect("read test dir") + .map(|entry| entry.expect("dir entry").file_name()) + .collect::>(); + entries.sort(); + entries + }; + + file_image( + "copy-to-image", + format!("{},boot:/my-file", in_file.to_string_lossy()), + ); + let entries = dir_entries(); + file_image( + "copy-from-image", + format!("boot:/my-file,{}", out_file.to_string_lossy()), + ); + + assert_eq!( + fs::read(&out_file).expect("read out file"), + fs::read(&in_file).expect("read in file") + ); + let mut expected = entries; + expected.push(out_file.file_name().expect("file name").to_owned()); + expected.sort(); + assert_eq!(dir_entries(), expected); +} + #[tokio::test] async fn check_ssh_tunnel_setup() { let tr = Testrunner::new("check_ssh_tunnel_setup");