From 4e71110a176f56f7f04236cf1f5c8e2478b2ba7b Mon Sep 17 00:00:00 2001 From: dkwolf1 <102389032+dkwolf1@users.noreply.github.com> Date: Tue, 10 Nov 2026 19:13:43 +0100 Subject: [PATCH] Harden Windows and WSL project backups --- CHANGELOG.md | 9 + build.ps1 | 2 +- docs/RELEASE-CHECKLIST.md | 24 +- docs/nl/RELEASE-CHECKLIST.md | 24 +- docs/nl/releases/v3.4.13.md | 25 ++ docs/releases/v3.4.13.md | 23 ++ pyproject.toml | 2 +- src/codex_transfer/__init__.py | 2 +- src/codex_transfer/backup.py | 447 ++++++++++++++++++++----- src/codex_transfer/gui.py | 43 ++- src/codex_transfer/path_model.py | 22 ++ src/codex_transfer/project_identity.py | 18 +- src/codex_transfer/restore.py | 61 +++- src/codex_transfer/restore_plan.py | 3 + src/codex_transfer/selftest.py | 138 ++++++++ version_info.txt | 8 +- 16 files changed, 719 insertions(+), 132 deletions(-) create mode 100644 docs/nl/releases/v3.4.13.md create mode 100644 docs/releases/v3.4.13.md diff --git a/CHANGELOG.md b/CHANGELOG.md index e46437c..9baab44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## Unreleased +## 3.4.13 - 2026-11-10 + +- Review unsafe project roots and report unreadable project paths before copying. +- Add optional, reported exclusions for project entries named `node_modules` + and `.venv`. Show nested roots and merge equivalent WSL UNC aliases. +- Recognize junctions and WSL links, preserve readable file data when metadata + copying is unsupported, and retry short Windows file locks. +- Keep Codex-managed installer packages local through backup and restore. + ## 3.4.12 - 2026-11-09 - Fix Windows backup preview and copying when a project contains a reserved device diff --git a/build.ps1 b/build.ps1 index fb9f147..e26b5ad 100644 --- a/build.ps1 +++ b/build.ps1 @@ -1,6 +1,6 @@ [CmdletBinding()] param( - [string]$Version = '3.4.12', + [string]$Version = '3.4.13', [string]$PythonPath, [switch]$SkipTests ) diff --git a/docs/RELEASE-CHECKLIST.md b/docs/RELEASE-CHECKLIST.md index d2b26c7..203131d 100644 --- a/docs/RELEASE-CHECKLIST.md +++ b/docs/RELEASE-CHECKLIST.md @@ -2,31 +2,31 @@ [Nederlands](nl/RELEASE-CHECKLIST.md) -This checklist publishes Codex Lifeboat 3.4.12 as a **stable release**. The tag +This checklist publishes Codex Lifeboat 3.4.13 as a **stable release**. The tag workflow builds and validates fresh release assets, creates their checksums and provenance attestations, and publishes the GitHub release. ## Before pushing -- Confirm that `git status --short` contains only intended 3.4.12 changes. +- Confirm that `git status --short` contains only intended 3.4.13 changes. - Confirm that no real backup, database, token, `.env`, EXE, ZIP, or build directory is staged. - Keep the generated `dist` directory locally; `.gitignore` deliberately prevents it from entering the repository. -- Do not create the `v3.4.12` tag until the `main` CI run is green. +- Do not create the `v3.4.13` tag until the `main` CI run is green. ## Publish with PowerShell Run these commands from the repository directory: ```powershell -git switch -c dkw1/fix-issue-13 +git switch -c dkw1/fix-issue-15 git add -u -git add -- docs/releases/v3.4.12.md docs/nl/releases/v3.4.12.md +git add -- docs/releases/v3.4.13.md docs/nl/releases/v3.4.13.md git diff --cached --check -git commit -m "Fix Windows backup preview and copy edge cases" -git push -u origin dkw1/fix-issue-13 -gh pr create --base main --head dkw1/fix-issue-13 --title "Fix Windows backup preview and copy edge cases" --body "Addresses issue #13." +git commit -m "Harden Windows and WSL project backups" +git push -u origin dkw1/fix-issue-15 +gh pr create --base main --head dkw1/fix-issue-15 --title "Harden Windows and WSL project backups" --body "Addresses issue #15." ``` Merge the PR after its checks pass, then wait until the `main` CI run is green at @@ -35,15 +35,15 @@ Merge the PR after its checks pass, then wait until the `main` CI run is green a ```powershell git switch main git pull --ff-only origin main -git tag -a v3.4.12 -m "Codex Lifeboat 3.4.12 stable release" -git push origin v3.4.12 +git tag -a v3.4.13 -m "Codex Lifeboat 3.4.13 stable release" +git push origin v3.4.13 ``` The tag triggers `.github/workflows/release.yml`. Do **not** also create a manual release with the same tag: the workflow creates it as the stable latest release. Follow it at , then -open . +open . ## Verify the published release @@ -68,7 +68,7 @@ gh attestation verify .\Codex-Lifeboat-Windows-x64-Portable.zip --repo dkwolf1/C ## Stable-release evidence -The release workflow runs the 99-check source/EXE/ZIP gate, including 12 automated +The release workflow runs the 101-check source/EXE/ZIP gate, including 12 automated Windows scenarios. A prior physical Windows 11 PC A → PC B → PC A workflow included both verify operations and opening and continuing work in Codex after restore. Windows 10 and USB-interruption coverage remain documented as pending. Public diff --git a/docs/nl/RELEASE-CHECKLIST.md b/docs/nl/RELEASE-CHECKLIST.md index a93b26f..d69fcb3 100644 --- a/docs/nl/RELEASE-CHECKLIST.md +++ b/docs/nl/RELEASE-CHECKLIST.md @@ -2,29 +2,29 @@ [English](../RELEASE-CHECKLIST.md) -Deze checklist publiceert Codex Lifeboat 3.4.12 als **stabiele release**. De +Deze checklist publiceert Codex Lifeboat 3.4.13 als **stabiele release**. De tag-workflow bouwt en controleert nieuwe releasebestanden, maakt de checksums en herkomstverklaringen en publiceert de GitHub-release. ## Vóór het pushen -- Controleer dat `git status --short` alleen bedoelde 3.4.12-wijzigingen bevat. +- Controleer dat `git status --short` alleen bedoelde 3.4.13-wijzigingen bevat. - Stage geen echte back-up, database, token, `.env`, EXE, ZIP of bouwmap. - De lokale map `dist` hoort niet in Git en wordt bewust genegeerd. -- Maak tag `v3.4.12` pas nadat de CI-run van `main` groen is. +- Maak tag `v3.4.13` pas nadat de CI-run van `main` groen is. ## Publiceren met PowerShell Voer in de repositorymap uit: ```powershell -git switch -c dkw1/fix-issue-13 +git switch -c dkw1/fix-issue-15 git add -u -git add -- docs/releases/v3.4.12.md docs/nl/releases/v3.4.12.md +git add -- docs/releases/v3.4.13.md docs/nl/releases/v3.4.13.md git diff --cached --check -git commit -m "Fix Windows backup preview and copy edge cases" -git push -u origin dkw1/fix-issue-13 -gh pr create --base main --head dkw1/fix-issue-13 --title "Fix Windows backup preview and copy edge cases" --body "Addresses issue #13." +git commit -m "Harden Windows and WSL project backups" +git push -u origin dkw1/fix-issue-15 +gh pr create --base main --head dkw1/fix-issue-15 --title "Harden Windows and WSL project backups" --body "Addresses issue #15." ``` Merge de pull request nadat de controles slagen. Wacht daarna tot de `main`-CI groen is op @@ -33,15 +33,15 @@ Merge de pull request nadat de controles slagen. Wacht daarna tot de `main`-CI g ```powershell git switch main git pull --ff-only origin main -git tag -a v3.4.12 -m "Codex Lifeboat 3.4.12 stable release" -git push origin v3.4.12 +git tag -a v3.4.13 -m "Codex Lifeboat 3.4.13 stable release" +git push origin v3.4.13 ``` De tag start `.github/workflows/release.yml`. Maak niet daarnaast handmatig een release met dezelfde tag: de workflow maakt deze als stabiele nieuwste release. Volg de uitvoering op en open -daarna . +daarna . ## Gepubliceerde release controleren @@ -64,7 +64,7 @@ Controleer na downloaden eventueel de GitHub-herkomstverklaring: gh attestation verify .\Codex-Lifeboat-Windows-x64-Portable.zip --repo dkwolf1/Codex-Lifeboat ``` -De releaseworkflow voert 99 bron-/EXE-/ZIP-controles uit, waaronder 12 +De releaseworkflow voert 101 bron-/EXE-/ZIP-controles uit, waaronder 12 geautomatiseerde Windows-scenario's. Een eerdere fysieke Windows 11-route van pc A naar pc B en terug bevatte beide controles en verder werken in Codex na herstel. Windows 10 en USB-onderbreking diff --git a/docs/nl/releases/v3.4.13.md b/docs/nl/releases/v3.4.13.md new file mode 100644 index 0000000..3483d75 --- /dev/null +++ b/docs/nl/releases/v3.4.13.md @@ -0,0 +1,25 @@ +# Codex Lifeboat 3.4.13 — projectback-ups op Windows en WSL + +[English](../../releases/v3.4.13.md) + +Deze update pakt de back-upfouten uit [issue #15](https://github.com/dkwolf1/Codex-Lifeboat/issues/15) aan. + +- De voorvertoning markeert te brede roots en toont geneste projecten. + De twee WSL-UNC-schrijfwijzen voor hetzelfde pad worden samengevoegd. + Geneste projectroots blijven apart selecteerbaar. +- In de projectselectie kun je `node_modules` en `.venv` uitsluiten. + Het back-uprapport vermeldt de overgeslagen paden. De getoonde grootte + is een schatting vóór deze uitsluitingen. +- De leescontrole vindt geblokkeerde projectpaden vóór het kopiëren. + De gevonden problemen staan in `reports/project-read-errors.json`. + Je kunt een getroffen project uitsluiten en zijn gesprekken meenemen. +- Lifeboat slaat herkende symlinks en junctions over en meldt dat in het + rapport. Bij een WSL-leesfout kan het programma het bestandstype in de + distro controleren. Gewone bestanden die onleesbaar blijven blokkeren + de back-up. +- Korte Windows-locks worden opnieuw geprobeerd. Als alleen het kopiëren + van de bestandsbytes lukt na `EINVAL`, meldt het rapport dat metadata + kan afwijken. +- De Codex-installaties onder `packages/standalone` en + `packages/app-server-daemon` blijven lokaal. Andere items onder + `packages` blijven onderdeel van de back-up. diff --git a/docs/releases/v3.4.13.md b/docs/releases/v3.4.13.md new file mode 100644 index 0000000..7f46d86 --- /dev/null +++ b/docs/releases/v3.4.13.md @@ -0,0 +1,23 @@ +# Codex Lifeboat 3.4.13 — safer project backups on Windows and WSL + +[Nederlands](../nl/releases/v3.4.13.md) + +This update addresses the project backup failures in [issue #15](https://github.com/dkwolf1/Codex-Lifeboat/issues/15). + +- The preview marks unsafe broad roots and shows when a project is nested + under another root. Equivalent WSL UNC spellings of the same root are copied + once. Separately registered nested roots remain selectable. +- The project selection screen offers opt-in exclusions for `node_modules` + and `.venv`. The backup report lists omitted entries. The size shown in + the preview is an estimate before these exclusions. +- The project preflight checks readability before copying payloads. When + required paths are blocked, `reports/project-read-errors.json` lists the + issues found. The selection screen allows users to exclude an affected + project while keeping its conversation history. +- Recognized symbolic links and junctions are reported and skipped. When a + WSL path fails to read, Lifeboat can ask the owning distro whether it is a + symbolic link. Regular files that remain unreadable block the backup. +- Copy operations retry short Windows locks. Where a byte copy succeeds + after `copy2` returns `EINVAL`, the report warns that metadata may differ. +- Codex's managed `packages/standalone` and `packages/app-server-daemon` + installations stay local. Other entries under `packages` remain portable. diff --git a/pyproject.toml b/pyproject.toml index 42924e9..7877b86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "codex-lifeboat" -version = "3.4.12" +version = "3.4.13" description = "One-click Codex backup, restore and Windows PC migration utility" readme = "README.md" license = "MIT" diff --git a/src/codex_transfer/__init__.py b/src/codex_transfer/__init__.py index 4cb9c26..a9b631c 100644 --- a/src/codex_transfer/__init__.py +++ b/src/codex_transfer/__init__.py @@ -1 +1 @@ -__version__ = "3.4.12" +__version__ = "3.4.13" diff --git a/src/codex_transfer/backup.py b/src/codex_transfer/backup.py index a5ebed9..702c1b6 100644 --- a/src/codex_transfer/backup.py +++ b/src/codex_transfer/backup.py @@ -30,7 +30,7 @@ FORMAT_ID = "codex-portable-backup" FORMAT_VERSION = "2.4" -GENERATOR_VERSION = "3.4.12" +GENERATOR_VERSION = "3.4.13" PROGRESS_CALLBACK = None STATUS_CALLBACK = None _RUNTIME_EXECUTABLE: Path | None = None @@ -63,6 +63,13 @@ } ) APP_SERVER_CONTROL_DIR = "app-server-control" +CODEX_INSTALLER_PACKAGE_NAMES = frozenset({"standalone", "app-server-daemon"}) +GENERATED_PROJECT_NAMES = frozenset({"node_modules", ".venv"}) +LINK_REPARSE_TAGS = { + stat.IO_REPARSE_TAG_SYMLINK, + stat.IO_REPARSE_TAG_MOUNT_POINT, + 0xA000001D, # IO_REPARSE_TAG_LX_SYMLINK +} BLOCKED_CODEX_DIRS = { ".sandbox", ".sandbox-bin", @@ -412,13 +419,23 @@ def portable_relative(path: Path, root: Path) -> str: def clean_windows_path(value: str) -> Path: expanded = os.path.expandvars(value).strip().strip('"') - if expanded.startswith("\\\\?\\"): + if expanded[:8].lower() == "\\\\?\\unc\\": + expanded = "\\\\" + expanded[8:] + elif expanded.startswith("\\\\?\\"): expanded = expanded[4:] return Path(expanded).expanduser() def normalized_source_key(path: Path) -> str: - return os.path.normcase(os.path.normpath(str(path.resolve(strict=False)))) + wsl_key = path_model.wsl_path_key(str(path)) + if wsl_key: + return wsl_key + try: + resolved = path.resolve(strict=False) + except OSError: + # A saved offline path still needs an identity for selection and reporting. + resolved = path + return os.path.normcase(os.path.normpath(str(resolved))) def matches_database_rollout_path(source: Path, database_path: str) -> bool: @@ -617,6 +634,81 @@ def copy_file(source: Path, destination: Path) -> None: shutil.copy2(source, destination) +def is_link_metadata(info: os.stat_result) -> bool: + return ( + stat.S_ISLNK(info.st_mode) + or getattr(info, "st_reparse_tag", None) in LINK_REPARSE_TAGS + ) + + +def confirms_wsl_link(path: Path) -> bool: + """Ask the owning distro about an entry Windows could not read.""" + aliases = path_model.wsl_path_aliases(str(path)) + if not aliases: + return False + distro, _, relative = aliases[0][len("\\\\wsl$\\"):].partition("\\") + if not relative or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", distro): + return False + system_root = os.environ.get("SystemRoot") + if not system_root: + return False + wsl_executable = Path(system_root) / "System32" / "wsl.exe" + linux_path = "/" + relative.replace("\\", "/") + try: + result = subprocess.run( + [str(wsl_executable), "--distribution", distro, "--exec", "/usr/bin/test", "-L", linux_path], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=5, + check=False, + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), + ) + except (OSError, subprocess.TimeoutExpired): + return False + return result.returncode == 0 + + +def entry_metadata(entry: os.DirEntry) -> os.stat_result: + """Read directory-entry metadata without following junctions or WSL links.""" + return entry.stat(follow_symlinks=False) + + +def copy_backup_payload(source: Path, destination: Path, warnings: list[str]) -> None: + """Retry transient Windows locks and preserve bytes if copy2 rejects metadata.""" + for attempt in range(4): + try: + copy_file(source, destination) + return + except OSError as exc: + if ( + getattr(exc, "winerror", None) in {5, 32, 33} + and attempt < 3 + ): + time.sleep(0.05 * (2**attempt)) + continue + if exc.errno == errno.EINVAL: + source_info = source.lstat() + if stat.S_ISREG(source_info.st_mode) and not is_link_metadata(source_info): + # A successful data-only copy is still verified in the hash manifest. + shutil.copyfile(source, destination) + warnings.append(f"Copied file data without preserving all metadata: {source}") + return + raise + + +def probe_project_file(path: Path) -> None: + for attempt in range(4): + try: + with path.open("rb") as stream: + stream.read(1) + return + except OSError as exc: + if getattr(exc, "winerror", None) not in {5, 32, 33} or attempt == 3: + raise + time.sleep(0.05 * (2**attempt)) + + def is_reconstructable_python_cache(path: Path) -> bool: """Return true only for disposable Python bytecode cache files.""" return bool( @@ -780,11 +872,14 @@ def dependency_preflight(source_codex: Path) -> dict[str, Any]: try: if entry.is_symlink(): continue - if entry.is_dir(follow_symlinks=False): + info = entry_metadata(entry) + if is_link_metadata(info): + continue + if stat.S_ISDIR(info.st_mode): path_type = "directory" stack.append(path) continue - if not entry.is_file(follow_symlinks=False): + if not stat.S_ISREG(info.st_mode): continue path_type = "file" scanned_files += 1 @@ -918,6 +1013,8 @@ def copy_tree( before_copy: Callable[[Path, Path, int], None] | None = None, *, allow_windows_device_skips: bool = True, + generated_names: frozenset[str] = frozenset(), + installer_packages: bool = False, ) -> tuple[int, int]: count = 0 byte_count = 0 @@ -925,9 +1022,7 @@ def copy_tree( stack: list[tuple[Path, Path]] = [(source, destination)] while stack: current_source, current_destination = stack.pop() - approved = (dependency_exclusions or {}).get( - normalized_source_key(current_source) - ) + approved = dependency_exclusions.get(normalized_source_key(current_source)) if dependency_exclusions else None if approved and approved.get("pathType") == "directory": if applied_dependency_exclusions is None: raise BackupError("Approved dependency exclusion has no audit record.") @@ -958,12 +1053,16 @@ def copy_tree( entry_destination = current_destination / entry.name confirmed_file = False relative_text = str(entry_source.relative_to(source)).lower() + if entry.name.lower() in generated_names: + warnings.append(f"Excluded by project dependency selection: {entry_source}") + continue + if installer_packages and relative_text in CODEX_INSTALLER_PACKAGE_NAMES: + warnings.append(f"Skipped Codex installer package: {entry_source}") + continue if any(fragment and fragment in relative_text for fragment in excluded): warnings.append(f"Excluded by configuration: {entry_source}") continue - approved = (dependency_exclusions or {}).get( - normalized_source_key(entry_source) - ) + approved = dependency_exclusions.get(normalized_source_key(entry_source)) if dependency_exclusions else None if approved and approved.get("pathType") == "file": if applied_dependency_exclusions is None: raise BackupError("Approved dependency exclusion has no audit record.") @@ -981,19 +1080,33 @@ def copy_tree( if entry.is_symlink(): warnings.append(f"Symbolic link or reparse point skipped: {entry_source}") continue - if entry.is_dir(follow_symlinks=False): + info = entry_metadata(entry) + if is_link_metadata(info): + warnings.append(f"Symbolic link or reparse point skipped: {entry_source}") + continue + if stat.S_ISDIR(info.st_mode): stack.append((entry_source, entry_destination)) - elif entry.is_file(follow_symlinks=False): + elif stat.S_ISREG(info.st_mode): confirmed_file = True - size = entry.stat(follow_symlinks=False).st_size + size = info.st_size if before_copy: before_copy(entry_source, entry_destination, size) - copy_file(entry_source, entry_destination) + if allow_windows_device_skips: + copy_backup_payload(entry_source, entry_destination, warnings) + else: + copy_file(entry_source, entry_destination) count += 1 byte_count += size if count % 1000 == 0: log(f" ... copied {count} files from {source.name}") + else: + warnings.append(f"Skipped non-regular filesystem entry: {entry_source}") except OSError as exc: + if allow_windows_device_skips and confirms_wsl_link(entry_source): + if confirmed_file: + entry_destination.unlink(missing_ok=True) + warnings.append(f"Unsupported WSL symbolic link skipped: {entry_source}") + continue if confirmed_file and record_skipped_reconstructable_cache( entry_source, entry_destination, @@ -1012,9 +1125,14 @@ def tree_stats( *, package_prefix: str | None = None, path_budget: dict[str, Any] | None = None, + generated_names: frozenset[str] = frozenset(), + installer_packages: bool = False, + read_issues: list[dict[str, str]] | None = None, ) -> tuple[int, int]: count, byte_count, _ = tree_stats_with_breakdown( - source, exclude_fragments, package_prefix=package_prefix, path_budget=path_budget + source, exclude_fragments, package_prefix=package_prefix, path_budget=path_budget, + generated_names=generated_names, installer_packages=installer_packages, + read_issues=read_issues, ) return count, byte_count @@ -1025,6 +1143,9 @@ def tree_stats_with_breakdown( *, package_prefix: str | None = None, path_budget: dict[str, Any] | None = None, + generated_names: frozenset[str] = frozenset(), + installer_packages: bool = False, + read_issues: list[dict[str, str]] | None = None, ) -> tuple[int, int, list[dict[str, Any]]]: """Return logical file usage and immediate-child totals without changing data.""" count = 0 @@ -1033,42 +1154,70 @@ def tree_stats_with_breakdown( excluded = [value.replace("/", os.sep).lower() for value in exclude_fragments] if path_budget is not None and package_prefix: record_projected_path(path_budget, source, package_prefix) - for root, directories, files in os.walk(source, followlinks=False): - root_path = Path(root) - kept_directories: list[str] = [] - for name in directories: - directory = root_path / name - if is_windows_device_name(name): - log(windows_device_skip_warning(directory)) - continue - relative = directory.relative_to(source) - if directory.is_symlink() or any( - fragment and fragment in str(relative).lower() for fragment in excluded - ): - continue - kept_directories.append(name) - if path_budget is not None and package_prefix: - record_projected_path( - path_budget, - directory, - PurePosixPath(package_prefix, *relative.parts).as_posix(), + stack = [source] + next_status = time.monotonic() + 0.5 + checked_files = 0 + while stack: + current = stack.pop() + try: + with os.scandir(current) as stream: + entries = list(stream) + except OSError as exc: + if read_issues is not None: + read_issues.append({"sourcePath": str(current), "error": str(exc)}) + continue + for entry in entries: + if read_issues is not None and time.monotonic() >= next_status: + report_status( + 0, 0, f"Checking project readability: {source.name} " + f"({checked_files:,} files checked, {len(read_issues):,} blocked paths)" ) - directories[:] = kept_directories - for name in files: - path = root_path / name - if is_windows_device_name(name): + next_status = time.monotonic() + 0.5 + path = current / entry.name + if is_windows_device_name(entry.name): log(windows_device_skip_warning(path)) continue - if path.is_symlink(): - continue relative = path.relative_to(source) relative_text = str(relative).lower() + if entry.name.lower() in generated_names: + continue + if installer_packages and relative_text in CODEX_INSTALLER_PACKAGE_NAMES: + continue if any(fragment in relative_text for fragment in excluded if fragment): continue + confirmed_file = False try: - size = path.stat().st_size - except OSError: + if entry.is_symlink(): + continue + info = entry_metadata(entry) + if is_link_metadata(info): + continue + directory = stat.S_ISDIR(info.st_mode) + if not directory and not stat.S_ISREG(info.st_mode): + continue + confirmed_file = not directory + if read_issues is not None and not directory: + checked_files += 1 + probe_project_file(path) + except OSError as exc: + if ( + read_issues is not None + and not (confirmed_file and is_reconstructable_python_cache(path)) + and not confirms_wsl_link(path) + ): + read_issues.append({"sourcePath": str(path), "error": str(exc)}) continue + if path_budget is not None and package_prefix: + record_projected_path( + path_budget, + path, + PurePosixPath(package_prefix, *relative.parts).as_posix(), + reconstructable_cache=not directory and is_reconstructable_python_cache(path), + ) + if directory: + stack.append(path) + continue + size = info.st_size top = relative.parts[0] if len(relative.parts) > 1 else "(root files)" item = children.setdefault( top, {"name": top, "fileCount": 0, "totalBytes": 0} @@ -1077,13 +1226,6 @@ def tree_stats_with_breakdown( item["totalBytes"] += size count += 1 byte_count += size - if path_budget is not None and package_prefix: - record_projected_path( - path_budget, - path, - PurePosixPath(package_prefix, *relative.parts).as_posix(), - reconstructable_cache=is_reconstructable_python_cache(path), - ) return ( count, byte_count, @@ -1119,29 +1261,30 @@ def _portable_profile_stats( lowered in blocked_names or lowered in blocked_dirs or is_codex_provisioning_guard_name(entry.name) - or entry.is_symlink() ): continue + try: + info = entry.lstat() + except FileNotFoundError: + continue + if is_link_metadata(info): + continue package_prefix = f"codex/portable-profile/{entry.name}" - if entry.is_dir(): + if stat.S_ISDIR(info.st_mode): files, size = tree_stats( entry, [], package_prefix=package_prefix, path_budget=path_budget, + installer_packages=lowered == "packages", ) - elif entry.is_file(): - try: - files, size = 1, entry.stat().st_size - if path_budget is not None: - record_projected_path( - path_budget, - entry, - package_prefix, - reconstructable_cache=is_reconstructable_python_cache(entry), - ) - except OSError: - continue + elif stat.S_ISREG(info.st_mode): + files, size = 1, info.st_size + if path_budget is not None: + record_projected_path( + path_budget, entry, package_prefix, + reconstructable_cache=is_reconstructable_python_cache(entry), + ) else: continue total_files += files @@ -1183,14 +1326,15 @@ def build_backup_preview( projects: list[dict[str, Any]] = [] for index, item in enumerate(candidates, start=1): - source = clean_windows_path(str(item["sourcePath"])).resolve(strict=False) + source = clean_windows_path(str(item["sourcePath"])) project_path_budget = new_path_budget() report_status( index - 1, max(len(candidates), 1), f"Scanning project {index}/{len(candidates)}: {source.name or source}", ) - if source.is_dir() and not broad_or_unsafe_project_path(source, profile, codex_home): + problem = project_root_problem(source, profile, codex_home) + if problem is None: file_count, total_bytes, folders = tree_stats_with_breakdown( source, excludes, @@ -1199,11 +1343,13 @@ def build_backup_preview( ) else: file_count, total_bytes, folders = 0, 0, [] + log(f"Project root unavailable for selection: {source}: {problem}") projects.append( { "name": str(item.get("name") or source.name or "project"), "path": str(source), - "sourcePresent": source.is_dir(), + "sourcePresent": problem is None, + "rootProblem": problem, "fileCount": file_count, "totalBytes": total_bytes, "largestFolders": folders[:8], @@ -1213,6 +1359,17 @@ def build_backup_preview( } ) + available_roots = { + normalized_source_key(clean_windows_path(item["path"])): item["path"] + for item in projects if item["sourcePresent"] + } + for item in projects: + if item["sourcePresent"]: + key = normalized_source_key(clean_windows_path(item["path"])) + parents = [root for root in available_roots if key.startswith(root + os.sep)] + if parents: + item["nestedUnder"] = available_roots[max(parents, key=len)] + codex_path_budget = new_path_budget() codex_files = 1 codex_bytes = source_db.stat().st_size @@ -1633,6 +1790,12 @@ def add_project_candidate( "codexProjectIds": [], }, ) + if candidate["sourcePath"] != str(path) and path_model.wsl_path_key(str(path)): + spellings = candidate.setdefault("wslPathAliases", [candidate["sourcePath"]]) + if str(path) not in spellings: + spellings.append(str(path)) + if str(path).lower().startswith("\\\\wsl.localhost\\"): + candidate["sourcePath"] = str(path) if origin not in candidate["origins"]: candidate["origins"].append(origin) candidate["required"] = bool(candidate["required"] or required) @@ -1691,7 +1854,11 @@ def collect_project_candidates( if not cwd_value: continue cwd = clean_windows_path(str(cwd_value)) - if not cwd.is_dir(): + try: + if not cwd.is_dir(): + continue + except OSError: + # Stale session working directories are discovery hints, not required data. continue cwd_key = normalized_source_key(cwd) if any(cwd_key == root or cwd_key.startswith(root + os.sep) for root in existing_roots): @@ -1711,8 +1878,11 @@ def collect_project_candidates( continue anchor = cwd for parent in (cwd, *cwd.parents): - if (parent / ".git").exists(): - anchor = parent + try: + if (parent / ".git").exists(): + anchor = parent + break + except OSError: break add_project_candidate( candidates, @@ -1857,9 +2027,7 @@ def filter_snapshot_for_excluded_projects( def is_reparse_point(path: Path) -> bool: try: - return path.is_symlink() or bool( - getattr(os.path, "isjunction", lambda _path: False)(path) - ) + return is_link_metadata(path.lstat()) except OSError: return False @@ -1872,11 +2040,15 @@ def analyze_project_candidates(candidates: list[dict[str, Any]]) -> dict[str, An for item in candidates: source = clean_windows_path(str(item["sourcePath"])) key = normalized_source_key(source) + try: + present = source.is_dir() + except OSError: + present = False roots.append( { "originalPath": str(source), "normalizedPath": key, - "sourcePresent": source.is_dir(), + "sourcePresent": present, "reparsePoint": is_reparse_point(source), "origins": list(item.get("origins", [])), "codexProjectIds": list(item.get("codexProjectIds", [])), @@ -1952,10 +2124,14 @@ def copy_portable_codex_profile( if is_codex_provisioning_guard_name(entry.name): continue relative = f"codex/portable-profile/{entry.name}" - if entry.is_symlink(): + try: + info = entry.lstat() + except OSError as exc: + raise BackupError(f"Cannot inspect Codex profile entry: {entry}: {exc}") from exc + if is_link_metadata(info): warnings.append(f"Codex symbolic link or reparse point skipped: {entry}") continue - if entry.is_dir(): + if stat.S_ISDIR(info.st_mode): count, size = copy_tree( entry, package_root / relative, @@ -1964,12 +2140,13 @@ def copy_portable_codex_profile( skipped_reconstructable_cache, dependency_exclusions if lowered == ".chatgpt-projects" else None, applied_dependency_exclusions, + installer_packages=lowered == "packages", ) - elif entry.is_file(): + elif stat.S_ISREG(info.st_mode): destination = package_root / relative try: - copy_file(entry, destination) - count, size = 1, entry.stat().st_size + copy_backup_payload(entry, destination, warnings) + count, size = 1, info.st_size except OSError as exc: if record_skipped_reconstructable_cache( entry, @@ -1994,6 +2171,10 @@ def copy_portable_codex_profile( def broad_or_unsafe_project_path(path: Path, source_profile: Path, codex_home: Path) -> bool: + if path.anchor and path == Path(path.anchor): + return True + if path_model.wsl_path_aliases(str(path)): + return False resolved = path.resolve(strict=False) anchors = { Path(resolved.anchor).resolve(strict=False), @@ -2009,6 +2190,29 @@ def broad_or_unsafe_project_path(path: Path, source_profile: Path, codex_home: P return resolved in anchors +def project_root_problem(path: Path, source_profile: Path, codex_home: Path) -> str | None: + try: + if broad_or_unsafe_project_path(path, source_profile, codex_home): + return "unsafe-broad-root" + if not path.is_dir(): + return "missing-directory" + except OSError: + return "unreadable-directory" + return None + + +def write_project_read_issues(building: Path, issues: list[dict[str, str]]) -> None: + if issues: + report = building / "reports" / "project-read-errors.json" + write_json(report, {"reportVersion": 1, "count": len(issues), "items": issues}) + first = issues[0] + raise BackupError( + f"Project preflight found {len(issues)} blocked path(s). " + f"First: {first['sourcePath']} ({first['error']}). " + f"Full details: {report}" + ) + + def copy_projects( candidates: list[dict[str, Any]], package_root: Path, @@ -2019,6 +2223,8 @@ def copy_projects( exclude_fragments: list[str], warnings: list[str], skipped_reconstructable_cache: list[dict[str, Any]] | None = None, + *, + generated_names: frozenset[str] = frozenset(), ) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]: projects: list[dict[str, Any]] = [] mappings: list[dict[str, Any]] = [] @@ -2095,6 +2301,7 @@ def copy_projects( exclude_fragments, warnings, skipped_reconstructable_cache, + generated_names=generated_names, ) projects.append( { @@ -2779,6 +2986,12 @@ def load_config(path: Path) -> dict[str, Any]: raise BackupError("config.additionalPortablePaths moet een lijst zijn.") if not isinstance(config.get("excludedProjectPaths", []), list): raise BackupError("config.excludedProjectPaths must be a list.") + generated = config.get("excludeGeneratedProjectDirectories", []) + if not isinstance(generated, list) or any( + not isinstance(name, str) or name.lower() not in GENERATED_PROJECT_NAMES + for name in generated + ): + raise BackupError("excludeGeneratedProjectDirectories supports node_modules and .venv.") return config @@ -3003,19 +3216,59 @@ def build_backup(args: argparse.Namespace) -> Path: candidates, excluded_candidates = select_project_candidates( all_candidates, config.get("excludedProjectPaths", []) ) + safe_candidates: list[dict[str, Any]] = [] + root_issues: list[dict[str, str]] = [] + building_key = normalized_source_key(building) + for item in candidates: + source = clean_windows_path(item["sourcePath"]) + if item.get("wslPathAliases"): + warnings.append( + "Equivalent WSL paths backed up once: " + + " | ".join(item["wslPathAliases"]) + ) + problem = project_root_problem(source, source_profile, source_codex) + if problem == "unsafe-broad-root" and not item.get("required"): + excluded_candidates.append(item) + warnings.append(f"Unsafe broad project root excluded: {source}") + continue + if problem and (item.get("required") or problem != "missing-directory"): + root_issues.append({"sourcePath": str(source), "error": problem}) + if not problem: + source_key = normalized_source_key(source) + if building_key == source_key or building_key.startswith(source_key + os.sep): + root_issues.append({ + "sourcePath": str(source), + "error": "The backup destination is inside the project source.", + }) + safe_candidates.append(item) + candidates = safe_candidates + write_project_read_issues(building, root_issues) + selected_project_ids = { + project_id for item in candidates for project_id in item.get("codexProjectIds", []) + } + excluded_registrations = [ + { + **item, + "codexProjectIds": [ + project_id for project_id in item.get("codexProjectIds", []) + if project_id not in selected_project_ids + ], + } + for item in excluded_candidates + ] excluded_project_ids = filter_snapshot_for_excluded_projects( building / "codex" / "state.snapshot.sqlite", database_info, - excluded_candidates, + excluded_registrations, ) portable_state = filter_portable_state_for_excluded_projects( - portable_state, excluded_candidates + portable_state, excluded_registrations ) write_portable_state(building, portable_state) if excluded_candidates: warnings.append( f"{len(excluded_candidates)} project(s) intentionally excluded from project-file backup; " - "their conversations remain included as projectless history." + "their conversations remain included." ) project_analysis = analyze_project_candidates(candidates) if project_analysis["nestedRoots"]: @@ -3027,6 +3280,9 @@ def build_backup(args: argparse.Namespace) -> Path: f"{len(project_analysis['reparsePointRoots'])} project root reparse point(s) detected and inventoried." ) excludes = [str(item) for item in config.get("excludeDirectoryNames", [])] + generated_names = frozenset( + name.lower() for name in config.get("excludeGeneratedProjectDirectories", []) + ) current_phase = "dependency-preflight" log("Preflight: scanning every reachable Codex project dependency path...") @@ -3094,15 +3350,20 @@ def build_backup(args: argparse.Namespace) -> Path: "codex/portable-global-state.json", ) - for item in candidates: + project_read_issues: list[dict[str, str]] = [] + for index, item in enumerate(candidates, start=1): path = clean_windows_path(item["sourcePath"]) - if path.is_dir() and not broad_or_unsafe_project_path(path, source_profile, source_codex): + if project_root_problem(path, source_profile, source_codex) is None: + report_status(index - 1, max(len(candidates), 1), f"Checking project readability: {path.name}") estimated_bytes += tree_stats( path, excludes, package_prefix="projects/root-00000000-0000-0000-0000-000000000000", path_budget=projected_paths, + generated_names=generated_names, + read_issues=project_read_issues, )[1] + write_project_read_issues(building, project_read_issues) for extra, source, relative in configured_extra_sources(config): if not source.exists(): @@ -3266,6 +3527,7 @@ def build_backup(args: argparse.Namespace) -> Path: excludes, warnings, skipped_reconstructable_cache, + generated_names=generated_names, ) unresolved = unresolved_cwds(database_info["threads"], candidates) for cwd in unresolved: @@ -3522,6 +3784,7 @@ def build_backup(args: argparse.Namespace) -> Path: ), }, "selection": { + "excludedGeneratedProjectDirectories": sorted(generated_names), "includedProjectPaths": [str(item["sourcePath"]) for item in candidates], "excludedProjectPaths": [ str(item["sourcePath"]) for item in excluded_candidates @@ -3729,8 +3992,13 @@ def finalize_lineage(hash_rows: list[dict[str, Any]]) -> Path: report_status(0, 0, "All checks passed; completing the backup safely...") current_phase = "finalization" project_identity.save_registry(identity_registry_path, identity_registry) - os.replace(building, final_path) - lineage.save_state(lineage_state_path, lineage.state_from_manifest(lineage_manifest)) + atomic_io.replace_with_retry(building, final_path) + try: + lineage.save_state(lineage_state_path, lineage.state_from_manifest(lineage_manifest)) + except OSError as exc: + message = f"Backup published, but the local comparison baseline could not be updated: {exc}" + warnings.append(message) + log(message) log("") log("PASSED") log(f"Backup: {final_path}") @@ -3741,8 +4009,11 @@ def finalize_lineage(hash_rows: list[dict[str, Any]]) -> Path: report_status(1, 1, "Backup complete") return final_path except Exception as exc: - error_path.parent.mkdir(parents=True, exist_ok=True) - atomic_io.write_text(error_path, traceback.format_exc()) + try: + error_path.parent.mkdir(parents=True, exist_ok=True) + atomic_io.write_text(error_path, traceback.format_exc()) + except OSError as report_error: + log(f"Could not write backup error log: {report_error}") try: package_path = building / "manifest" / "package.json" if package_path.is_file(): diff --git a/src/codex_transfer/gui.py b/src/codex_transfer/gui.py index a484dc9..2cd25cf 100644 --- a/src/codex_transfer/gui.py +++ b/src/codex_transfer/gui.py @@ -107,6 +107,11 @@ def ask_backup_destination(parent, title: str, initial_directory: str) -> str: "selection_total": "Geselecteerd: {projects}/{total} projecten · {files} bestanden · ongeveer {size}", "selection_locked": "Altijd inbegrepen", "selection_missing": "Niet gevonden", + "selection_unsafe_root": "Te brede map", + "selection_unreadable_root": "Niet toegankelijk", + "selection_nested_root": "Genest onder {path}. Beide geselecteerde roots worden apart opgenomen.", + "selection_exclude_generated": "{name} uitsluiten in geselecteerde projecten", + "selection_generated_estimate": "De getoonde grootte is vóór deze uitsluitingen. De definitieve controle gebeurt vóór het kopiëren.", "selection_included": "Meegenomen", "selection_excluded": "Uitgesloten", "selection_codex_card": "Codex-gegevens beschermd", @@ -392,6 +397,11 @@ def ask_backup_destination(parent, title: str, initial_directory: str) -> str: "selection_total": "Selected: {projects}/{total} projects · {files} files · approximately {size}", "selection_locked": "Always included", "selection_missing": "Not found", + "selection_unsafe_root": "Root too broad", + "selection_unreadable_root": "Inaccessible", + "selection_nested_root": "Nested under {path}. Both selected roots are included separately.", + "selection_exclude_generated": "Exclude {name} in selected projects", + "selection_generated_estimate": "The size shown is before these exclusions. The final check runs before copying.", "selection_included": "Included", "selection_excluded": "Excluded", "selection_codex_card": "Codex data protected", @@ -1431,6 +1441,11 @@ def __init__(self, parent: "TransferApp", preview: dict) -> None: if item.get("sourcePresent") } self.project_by_iid: dict[str, dict] = {} + self.generated_choices = { + name: tk.BooleanVar(master=self, value=False) + for name in sorted(backup.GENERATED_PROJECT_NAMES) + } + self.excluded_generated_names: list[str] = [] self.title(parent.t("selection_title")) screen_width = max(parent.winfo_screenwidth(), 1100) screen_height = max(parent.winfo_screenheight(), 720) @@ -1662,6 +1677,18 @@ def _build(self) -> None: font=("Segoe UI", 9), wraplength=1100, justify="left", ) self.details_label.pack(anchor="w") + generated_options = ttk.Frame(details_card, style="App.TFrame") + generated_options.pack(anchor="w", pady=(6, 0)) + for name, selected in self.generated_choices.items(): + ttk.Checkbutton( + generated_options, + text=self.parent_app.t("selection_exclude_generated", name=name), + variable=selected, + ).pack(side="left", padx=(0, 14)) + ttk.Label( + details_card, text=self.parent_app.t("selection_generated_estimate"), + style="Body.TLabel", + ).pack(anchor="w") footer = ttk.Frame(self, style="App.TFrame") footer.grid(row=5, column=0, sticky="ew", padx=26, pady=(8, 22)) @@ -1722,7 +1749,10 @@ def _refresh(self) -> None: for iid, item in self.project_by_iid.items(): path = str(item["path"]) state = ( - self.parent_app.t("selection_missing") + self.parent_app.t({ + "unsafe-broad-root": "selection_unsafe_root", + "unreadable-directory": "selection_unreadable_root", + }.get(item.get("rootProblem"), "selection_missing")) if not item.get("sourcePresent") else "✓ " + self.parent_app.t("selection_included") if path in self.selected_paths @@ -1783,6 +1813,9 @@ def _show_details(self, _event=None) -> None: for folder in folders[:6] ) detail = "\n".join(lines) + if item.get("nestedUnder"): + nested = self.parent_app.t("selection_nested_root", path=item["nestedUnder"]) + detail = f"{nested}\n{detail}" self.details_label.configure(text=detail) def _approve(self) -> None: @@ -1794,6 +1827,9 @@ def _approve(self) -> None: ): return self.approved = True + self.excluded_generated_names = [ + name for name, selected in self.generated_choices.items() if selected.get() + ] self.destroy() @@ -3481,7 +3517,9 @@ def _continue_backup_from_preview(self, preview: dict) -> None: return path_report = backup.destination_path_budget( Path(selected), - backup.preview_path_budget_items(preview, dialog.excluded_paths), + (preview.get("codex") or {}).get("pathBudgetItems", []) + if dialog.excluded_generated_names + else backup.preview_path_budget_items(preview, dialog.excluded_paths), ) if path_report["safe"]: break @@ -3546,6 +3584,7 @@ def work(): "excludedProjectPaths": dialog.excluded_paths, "additionalPortablePaths": [], "excludeDirectoryNames": [], + "excludeGeneratedProjectDirectories": dialog.excluded_generated_names, "approvedDependencyExclusions": [ { "sourcePath": item.get("sourcePath"), diff --git a/src/codex_transfer/path_model.py b/src/codex_transfer/path_model.py index 93a4b8a..565bf33 100644 --- a/src/codex_transfer/path_model.py +++ b/src/codex_transfer/path_model.py @@ -29,6 +29,28 @@ class LocationError(ValueError): pass +def wsl_path_aliases(value: str) -> tuple[str, str] | None: + """Return the two UNC spellings while preserving Linux path component case.""" + path = str(value).replace("/", "\\") + if path[:8].lower() == "\\\\?\\unc\\": + path = "\\\\" + path[8:] + if not path.startswith("\\\\"): + return None + host, _, suffix = path[2:].partition("\\") + if host.lower() not in {"wsl$", "wsl.localhost"} or not suffix: + return None + return "\\\\wsl$\\" + suffix, "\\\\wsl.localhost\\" + suffix + + +def wsl_path_key(value: str) -> str | None: + aliases = wsl_path_aliases(ntpath.normpath(value)) + if not aliases: + return None + prefix = "\\\\wsl$\\" + distro, separator, suffix = aliases[0][len(prefix):].partition("\\") + return (prefix + distro.lower() + separator + suffix).rstrip("\\") + + def _path(value: str) -> PureWindowsPath: normalized = ntpath.normpath(str(value).strip()) candidate = PureWindowsPath(normalized) diff --git a/src/codex_transfer/project_identity.py b/src/codex_transfer/project_identity.py index 317e45d..c4e12d1 100644 --- a/src/codex_transfer/project_identity.py +++ b/src/codex_transfer/project_identity.py @@ -11,7 +11,7 @@ from pathlib import Path from typing import Any, Iterable -from . import atomic_io +from . import atomic_io, path_model REGISTRY_VERSION = 1 @@ -27,7 +27,7 @@ def utc_now() -> str: def canonical_path(value: str | Path) -> str: - return ntpath.normcase(ntpath.normpath(str(value))).casefold() + return path_model.wsl_path_key(str(value)) or ntpath.normcase(ntpath.normpath(str(value))).casefold() def empty_registry() -> dict[str, Any]: @@ -106,9 +106,9 @@ def _require_valid(value: Any) -> None: def _git_remote_hashes(project_path: Path) -> list[str]: config = project_path / ".git" / "config" - if not config.is_file(): - return [] try: + if not config.is_file(): + return [] text = config.read_text(encoding="utf-8-sig", errors="replace") except OSError: return [] @@ -121,10 +121,14 @@ def _git_remote_hashes(project_path: Path) -> list[str]: def identity_evidence(project_path: Path) -> dict[str, Any]: + try: + has_git_metadata = (project_path / ".git").exists() + except OSError: + has_git_metadata = False return { "pathKey": canonical_path(project_path), "gitRemoteHashes": _git_remote_hashes(project_path), - "hasGitMetadata": (project_path / ".git").exists(), + "hasGitMetadata": has_git_metadata, } @@ -176,7 +180,9 @@ def assign_identity( if set(project.get("codexProjectIds", [])) & set(codex_ids): codex_matches.append(project) for root in project.get("roots", []): - if evidence["pathKey"] in root.get("pathKeys", []): + previous_wsl_key = path_model.wsl_path_key(str(root.get("lastKnownPath", ""))) + known_keys = [previous_wsl_key] if previous_wsl_key else root.get("pathKeys", []) + if evidence["pathKey"] in known_keys: path_matches.append((project, root)) if evidence["gitRemoteHashes"] and set(evidence["gitRemoteHashes"]) & set( root.get("gitRemoteHashes", []) diff --git a/src/codex_transfer/restore.py b/src/codex_transfer/restore.py index 25d5426..eb44640 100644 --- a/src/codex_transfer/restore.py +++ b/src/codex_transfer/restore.py @@ -6,6 +6,7 @@ import os import shutil import sqlite3 +import stat import traceback import uuid from pathlib import Path, PurePosixPath @@ -29,7 +30,7 @@ from validate import validate -RESTORE_VERSION = "3.4.12" +RESTORE_VERSION = "3.4.13" PROTECTED_NAMES = {"auth.json", "installation_id", "cap_sid"} RUNTIME_NAMES = backup.BLOCKED_CODEX_DIRS PORTABLE_STATE_KEYS = backup.PORTABLE_STATE_KEYS @@ -139,7 +140,9 @@ def plan_restore_locations( ) -def _copy_all(source: Path, destination: Path) -> tuple[int, int]: +def _copy_all( + source: Path, destination: Path, *, installer_packages: bool = False +) -> tuple[int, int]: warnings: list[str] = [] files_since_check = 0 bytes_since_check = 0 @@ -164,6 +167,7 @@ def check_space(_source: Path, target: Path, size: int) -> None: warnings, before_copy=check_space, allow_windows_device_skips=False, + installer_packages=installer_packages, ) @@ -266,7 +270,7 @@ def _copy_codex_safety(source_codex: Path, safety_codex: Path) -> None: if entry.is_symlink(): continue if entry.is_dir(): - _copy_all(entry, destination) + _copy_all(entry, destination, installer_packages=lowered == "packages") elif entry.is_file(): backup.copy_file(entry, destination) database = source_codex / "state_5.sqlite" @@ -418,6 +422,8 @@ def __init__( target = self._project_target(original, project_id, item.get("location")) self.project_targets[project_id] = target self._add_replacement(str(original), str(target)) + for alias in path_model.wsl_path_aliases(str(original)) or (): + self._add_replacement(alias, str(target)) for item in self.mappings.get("attachments", []): attachment_id = str(item["id"]) name = Path(item.get("originalPath", "attachment.bin")).name @@ -1165,6 +1171,7 @@ def _restore_portable_profile( count = 0 if not source_root.is_dir(): return count + entries: list[tuple[Path, Path]] = [] for entry in source_root.iterdir(): destination = target_codex / entry.name # This database is a projection of rollout bytes. Older Lifeboat @@ -1176,6 +1183,18 @@ def _restore_portable_profile( continue if backup.is_codex_provisioning_guard_name(entry.name): continue + if entry.name.lower() == "packages": + if entry.is_dir() and not backup.is_link_metadata(entry.lstat()): + children = [ + child for child in entry.iterdir() + if child.name.lower() not in backup.CODEX_INSTALLER_PACKAGE_NAMES + ] + if children and not _safe_packages_container(destination): + raise RestoreError(f"Unsafe Codex packages directory: {destination}") + entries.extend((child, destination / child.name) for child in children) + continue + entries.append((entry, destination)) + for entry, destination in entries: if destination.exists(): _remove_path(destination) if entry.is_dir(): @@ -1205,6 +1224,14 @@ def _restore_portable_profile( return count +def _safe_packages_container(path: Path) -> bool: + try: + metadata = path.lstat() + except FileNotFoundError: + return True + return stat.S_ISDIR(metadata.st_mode) and not backup.is_link_metadata(metadata) + + def _reset_paginated_history_projection(target_codex: Path) -> int: """Remove derived paginated history so Codex rematerializes rewritten rollouts.""" removed = 0 @@ -1454,9 +1481,33 @@ def rollback_restore(safety_root: Path, target_profile: Path) -> None: journal_path = safety_root / "restore-journal.json" journal = backup.read_json(journal_path) target_codex = target_profile / ".codex" - if target_codex.exists(): + saved_codex = safety_root / "codex-before-restore" + target_packages = target_codex / "packages" + keep_installer = ( + target_packages.is_dir() + and _safe_packages_container(target_packages) + and any( + entry.name.lower() in backup.CODEX_INSTALLER_PACKAGE_NAMES + for entry in target_packages.iterdir() + ) + ) + if keep_installer: + for entry in target_codex.iterdir(): + if entry.name.lower() != "packages": + _remove_path(entry) + for entry in target_packages.iterdir(): + if entry.name.lower() not in backup.CODEX_INSTALLER_PACKAGE_NAMES: + _remove_path(entry) + elif target_codex.exists(): _remove_path(target_codex) - _copy_all(safety_root / "codex-before-restore", target_codex) + for entry in saved_codex.iterdir(): + target = target_codex / entry.name + if backup.is_link_metadata(entry.lstat()): + continue + if entry.is_dir(): + _copy_all(entry, target, installer_packages=entry.name.lower() == "packages") + elif entry.is_file(): + backup.copy_file(entry, target) approved_external_roots = tuple( Path(value).resolve(strict=False) for value in journal.get("approvedExternalRoots", []) diff --git a/src/codex_transfer/restore_plan.py b/src/codex_transfer/restore_plan.py index 7c95a6e..6a9960e 100644 --- a/src/codex_transfer/restore_plan.py +++ b/src/codex_transfer/restore_plan.py @@ -1062,6 +1062,9 @@ def build_restore_plan( for entry in codex_target.iterdir(): if entry.name.lower() == backup.APP_SERVER_CONTROL_DIR: continue + if entry.name.lower() == "packages" and entry.is_dir() and not entry.is_symlink(): + codex_target_bytes += backup.tree_stats(entry, [], installer_packages=True)[1] + continue if entry.is_file(): codex_target_bytes += entry.stat().st_size elif not entry.is_symlink() and entry.is_dir(): diff --git a/src/codex_transfer/selftest.py b/src/codex_transfer/selftest.py index 0a45918..53068f6 100644 --- a/src/codex_transfer/selftest.py +++ b/src/codex_transfer/selftest.py @@ -1517,6 +1517,7 @@ def reject_plugin_copy( skipped_reconstructable_cache: list[dict[str, Any]] | None = None, dependency_exclusions: dict[str, dict[str, Any]] | None = None, applied_dependency_exclusions: list[dict[str, Any]] | None = None, + **kwargs: Any, ) -> tuple[int, int]: if source.name.lower() == "plugins": raise PermissionError(87, "plugin runtime must not be opened", str(source)) @@ -1528,6 +1529,7 @@ def reject_plugin_copy( skipped_reconstructable_cache, dependency_exclusions, applied_dependency_exclusions, + **kwargs, ) warnings: list[str] = [] @@ -1747,6 +1749,140 @@ def text(value: str) -> str: ) +def _project_payload_preflight_test(root: Path) -> bool: + source = root / "project-payload-source" + source.mkdir() + for relative in ( + "code.py", "node_modules/.bin/playwright", ".venv/lib64", + "node_modules-notes/readme.txt", "blocked-one.txt", "blocked-two.txt", + "wsl-link", "junction/nested.txt", + ): + file = source / relative + file.parent.mkdir(parents=True, exist_ok=True) + file.write_bytes(relative.encode("utf-8")) + + original_probe = backup.probe_project_file + original_copy = backup.copy_file + original_stat = backup.entry_metadata + + def metadata(entry): + if entry.name == "junction": + return mock.Mock(st_mode=0o040000, st_reparse_tag=0xA0000003) + return original_stat(entry) + + def probe(path: Path): + if path.name.startswith("blocked-"): + raise PermissionError(13, "Injected unreadable user file", str(path)) + if path.name == "wsl-link": + raise FileNotFoundError(2, "Injected UNC link failure", str(path)) + original_probe(path) + + def copy(source: Path, target: Path): + if source.name == "wsl-link": + target.write_bytes(b"partial") + raise FileNotFoundError(2, "Injected UNC link failure", str(source)) + if source.name == "code.py": + raise OSError(22, "Injected copy2 metadata failure", str(source)) + original_copy(source, target) + + issues: list[dict[str, str]] = [] + warnings: list[str] = [] + generated_names = backup.GENERATED_PROJECT_NAMES + target = root / "project-payload-copy" + with ( + mock.patch.object(backup, "entry_metadata", side_effect=metadata), + mock.patch.object(backup, "probe_project_file", side_effect=probe), + mock.patch.object(backup, "copy_file", side_effect=copy), + mock.patch.object(backup, "confirms_wsl_link", side_effect=lambda path: path.name == "wsl-link"), + ): + backup.tree_stats(source, [], read_issues=issues, generated_names=generated_names) + files, _ = backup.copy_tree(source, target, [], warnings, generated_names=generated_names) + failure_report = root / "project-preflight-report" + try: + backup.write_project_read_issues(failure_report, issues) + except backup.BackupError: + blocked_report = backup.read_json(failure_report / "reports" / "project-read-errors.json") + else: + return False + return bool( + {Path(item["sourcePath"]).name for item in issues} == {"blocked-one.txt", "blocked-two.txt"} + and blocked_report["count"] == 2 + and files == 4 + and (target / "code.py").read_bytes() == b"code.py" + and (target / "node_modules-notes" / "readme.txt").is_file() + and not any((target / name).exists() for name in ("junction", "wsl-link", ".venv", "node_modules")) + and any("Unsupported WSL symbolic link skipped" in warning for warning in warnings) + and any("without preserving all metadata" in warning for warning in warnings) + and sum("project dependency selection" in warning for warning in warnings) == 2 + and backup.broad_or_unsafe_project_path(Path(source.anchor), source, source / ".codex") + ) + + +def _wsl_alias_and_installer_policy_test(root: Path) -> bool: + aliases = (r"\\wsl$\Ubuntu\home\user\Project", r"\\wsl.localhost\Ubuntu\home\user\Project") + candidates: dict[str, dict[str, Any]] = {} + for index, alias in enumerate(aliases): + backup.add_project_candidate(candidates, alias, "Project", "test", False, str(index)) + with ( + mock.patch.dict(os.environ, {"SystemRoot": r"C:\Windows"}), + mock.patch.object(backup.subprocess, "run", side_effect=[ + mock.Mock(returncode=0), mock.Mock(returncode=1), + ]) as run_wsl, + ): + wsl_link = backup.confirms_wsl_link(Path(aliases[0] + r"\lib64")) + wsl_regular_file = backup.confirms_wsl_link(Path(aliases[1] + r"\source.py")) + wsl_command = run_wsl.call_args_list[0].args[0] + source = root / "installer-policy-source" + package = root / "installer-policy-package" + target_profile = root / "installer-policy-target" + target = target_profile / ".codex" + for codex in (source, target): + installer = codex / "packages" / "standalone" + installer.mkdir(parents=True) + (installer / "current").write_bytes(b"local-installation") + (codex / "packages" / "custom.bin").write_bytes(b"custom-data") + warnings: list[str] = [] + files, _, _ = backup._portable_profile_stats(source) + backup.copy_portable_codex_profile(source, package, warnings) + portable = package / "codex" / "portable-profile" / "packages" + excluded = not (portable / "standalone").exists() + (portable / "standalone").mkdir() + (portable / "standalone" / "current").write_bytes(b"legacy-installer") + safety = root / "installer-policy-safety" + restore._copy_codex_safety(target, safety / "codex-before-restore") + backup.write_json(safety / "restore-journal.json", {"projects": []}) + safety_excluded = not (safety / "codex-before-restore" / "packages" / "standalone").exists() + (portable / "custom.bin").write_bytes(b"from-backup") + + class IdentityTranslator: + @staticmethod + def text(value: str) -> str: + return value + + restored = restore._restore_portable_profile(package, target, IdentityTranslator()) + installed_kept = (target / "packages" / "standalone" / "current").read_bytes() == b"local-installation" + restored_custom = (target / "packages" / "custom.bin").read_bytes() == b"from-backup" + restore.rollback_restore(safety, target_profile) + return bool( + len(candidates) == 1 + and next(iter(candidates.values()))["codexProjectIds"] == ["0", "1"] + and wsl_link + and not wsl_regular_file + and wsl_command[1:] == ["--distribution", "Ubuntu", "--exec", "/usr/bin/test", "-L", "/home/user/Project/lib64"] + and path_model.wsl_path_key(aliases[0]) != path_model.wsl_path_key(aliases[1].lower()) + and path_model.wsl_path_key(aliases[0]) != path_model.wsl_path_key(aliases[1].replace("Ubuntu", "Other")) + and files == restored == 1 + and excluded + and safety_excluded + and installed_kept + and restored_custom + and (target / "packages" / "standalone" / "current").read_bytes() == b"local-installation" + and (target / "packages" / "custom.bin").read_bytes() == b"custom-data" + and len(warnings) == 1 + and "Codex installer package" in warnings[0] + ) + + def _paginated_history_projection_policy_test(root: Path) -> bool: fixture = root / "paginated-history-projection-policy" source_codex = fixture / "source-user" / ".codex" @@ -4880,6 +5016,8 @@ def fail_error_journal(path: Path, value: Any) -> None: "provisioningGuardRuntimeExcluded": _provisioning_guard_policy_test(root), "reservedWindowsDevicePathsReported": _reserved_windows_device_path_test(root), "appServerControlRuntimeExcluded": _app_server_control_policy_test(root), + "projectPayloadPreflightAndRecovery": _project_payload_preflight_test(root), + "wslAliasesAndInstallerPackages": _wsl_alias_and_installer_policy_test(root), "projectRootInventoryAnalysis": _project_inventory_analysis_test(root), "lineageChangeModel": _lineage_model_test(), "streamingLineageDigest": _streaming_lineage_digest_test(root), diff --git a/version_info.txt b/version_info.txt index 3a1ebb9..fd691f7 100644 --- a/version_info.txt +++ b/version_info.txt @@ -1,7 +1,7 @@ VSVersionInfo( ffi=FixedFileInfo( - filevers=(3, 4, 12, 0), - prodvers=(3, 4, 12, 0), + filevers=(3, 4, 13, 0), + prodvers=(3, 4, 13, 0), mask=0x3f, flags=0x0, OS=0x40004, @@ -16,11 +16,11 @@ VSVersionInfo( [ StringStruct(u'CompanyName', u'Codex Lifeboat community project'), StringStruct(u'FileDescription', u'Codex Lifeboat'), - StringStruct(u'FileVersion', u'3.4.12'), + StringStruct(u'FileVersion', u'3.4.13'), StringStruct(u'InternalName', u'Codex-Lifeboat'), StringStruct(u'OriginalFilename', u'Codex-Lifeboat.exe'), StringStruct(u'ProductName', u'Codex Lifeboat'), - StringStruct(u'ProductVersion', u'3.4.12') + StringStruct(u'ProductVersion', u'3.4.13') ] ) ]),