Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions pkg/private/install.py.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ class NativeInstaller(object):
self.destdir = destdir
self.wipe_destdir = wipe_destdir
self.entries = []
self.tree_files = manifest.TreeFileTracker()

# Logger helper method, may not be necessary or desired
def _subst_destdir(path, self):
Expand Down Expand Up @@ -137,11 +138,19 @@ class NativeInstaller(object):
self._chown_chmod(entry.dest, entry.mode, entry.user, entry.group)

def _install_treeartifact_file(self, entry, src, dst):
if not self.tree_files.add_file(dst, src, entry.src):
return
self._do_file_copy(src, dst)
self._chown_chmod(dst, entry.mode, entry.user, entry.group)

def _install_treeartifact(self, entry):
logging.debug("COPYTREE %s <- %s/**", entry.dest, entry.src)
for root, dirs, _ in os.walk(entry.src):
rel_root = os.path.relpath(root, entry.src)
for d in dirs:
self.tree_files.add_dir(
os.path.normpath(os.path.join(entry.dest, rel_root, d)),
entry.src)
shutil.copytree(
src=entry.src,
dst=entry.dest,
Expand Down
41 changes: 41 additions & 0 deletions pkg/private/manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
"""Common package builder manifest helpers
"""

import filecmp
import json


Expand Down Expand Up @@ -79,3 +80,43 @@ def entry_type_to_string(et):
return "empty_file"
else:
raise ValueError("Invalid entry id {}".format(et))


class TreeFileTracker(object):
"""Detects conflicting paths provided by several tree artifacts."""

def __init__(self):
self._seen = {}

def add_file(self, dest, content_path, tree):
"""Records the file at dest, and returns whether it must be written.

Raises:
ValueError: dest was already provided by another tree as a directory
or as a file with different content.
"""
previous = self._seen.get(dest)
if previous is None:
self._seen[dest] = (content_path, tree)
return True
previous_path, previous_tree = previous
if previous_path is not None and filecmp.cmp(previous_path, content_path, shallow=False):
return False
raise ValueError(
"Conflicting path %s, provided by tree artifacts %s and %s" %
(dest, previous_tree, tree))

def add_dir(self, dest, tree):
"""Records the directory at dest.

Raises:
ValueError: dest was already provided by another tree as a file.
"""
dest = dest.rstrip("/")
previous = self._seen.get(dest)
if previous is None:
self._seen[dest] = (None, tree)
elif previous[0] is not None:
raise ValueError(
"Conflicting path %s, provided by tree artifacts %s and %s" %
(dest, previous[1], tree))
60 changes: 36 additions & 24 deletions pkg/private/pkg_files.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,12 @@ Concepts and terms:
When attributes are empty in DestFile, we let the package
tool decide their values.

content map: The map of destination paths to DestFile instances. Note that
content map: The map of destination paths to lists of DestFile instances. Note that
several distinct destinations make share the same source path.
Attempting to insert a duplicate entry in the content map is
an error, because it means you are collapsing files together.
We may want to relax this in the future if their DestFiles
are equal.
are equal. Only tree artifacts may share a destination.

manifest: The file which represents the content map. This is generated
by rule implementations and passed to the build_*.py helpers.
Expand Down Expand Up @@ -137,10 +137,21 @@ def create_mapping_context_from_ctx(
default_gid = None,
)

def _is_tree(entry):
return entry.src != None and entry.src.is_directory

def _add_entry(content_map, dest, entry):
old_entries = content_map.get(dest)
if old_entries and _is_tree(entry) and _is_tree(old_entries[0]):
content_map[dest] = [e for e in old_entries if e.src != entry.src] + [entry]
else:
content_map[dest] = [entry]

def _check_dest(content_map, dest, src, origin, allow_duplicates_with_different_content = False):
old_entry = content_map.get(dest)
if not old_entry:
old_entries = content_map.get(dest)
if not old_entries:
return
old_entry = old_entries[0]
if old_entry.src == src or old_entry.origin == origin:
return

Expand All @@ -158,7 +169,7 @@ def _check_dest(content_map, dest, src, origin, allow_duplicates_with_different_
msg = "Duplicate output path: <%s>, declared in %s and %s\n SRC: %s" % (
dest,
origin,
content_map[dest].origin,
old_entry.origin,
src,
)
if allow_duplicates_with_different_content:
Expand Down Expand Up @@ -204,7 +215,7 @@ def _process_pkg_dirs(mapping_context, pkg_dirs_info, origin):
for dir in pkg_dirs_info.dirs:
dest = dir.strip("/")
_check_dest(mapping_context.content_map, dest, None, origin, mapping_context.allow_duplicates_with_different_content)
mapping_context.content_map[dest] = _DestFile(
_add_entry(mapping_context.content_map, dest, _DestFile(
src = None,
entry_type = ENTRY_IS_DIR,
mode = attrs[0],
Expand All @@ -213,14 +224,14 @@ def _process_pkg_dirs(mapping_context, pkg_dirs_info, origin):
uid = attrs[3],
gid = attrs[4],
origin = origin,
)
))

def _process_pkg_files(mapping_context, pkg_files_info, origin):
attrs = _merge_context_attributes(pkg_files_info, mapping_context)
for filename, src in pkg_files_info.dest_src_map.items():
dest = filename.strip("/")
_check_dest(mapping_context.content_map, dest, src, origin, mapping_context.allow_duplicates_with_different_content)
mapping_context.content_map[dest] = _DestFile(
_add_entry(mapping_context.content_map, dest, _DestFile(
src = src,
entry_type = ENTRY_IS_TREE if src.is_directory else ENTRY_IS_FILE,
mode = attrs[0],
Expand All @@ -229,13 +240,13 @@ def _process_pkg_files(mapping_context, pkg_files_info, origin):
uid = attrs[3],
gid = attrs[4],
origin = origin,
)
))

def _process_pkg_symlink(mapping_context, pkg_symlink_info, origin):
dest = pkg_symlink_info.destination.strip("/")
attrs = _merge_context_attributes(pkg_symlink_info, mapping_context)
_check_dest(mapping_context.content_map, dest, None, origin, mapping_context.allow_duplicates_with_different_content)
mapping_context.content_map[dest] = _DestFile(
_add_entry(mapping_context.content_map, dest, _DestFile(
src = None,
entry_type = ENTRY_IS_LINK,
mode = attrs[0],
Expand All @@ -245,7 +256,7 @@ def _process_pkg_symlink(mapping_context, pkg_symlink_info, origin):
gid = attrs[4],
origin = origin,
link_to = pkg_symlink_info.target,
)
))

def _process_pkg_filegroup(mapping_context, pkg_filegroup_info):
if hasattr(pkg_filegroup_info, "pkg_dirs"):
Expand Down Expand Up @@ -317,7 +328,7 @@ def add_directory(mapping_context, dir_path, origin, mode = None, user = None, g
uid: numeric uid
gid: numeric gid
"""
mapping_context.content_map[dir_path.strip("/")] = _DestFile(
_add_entry(mapping_context.content_map, dir_path.strip("/"), _DestFile(
src = None,
entry_type = ENTRY_IS_DIR,
origin = origin,
Expand All @@ -326,7 +337,7 @@ def add_directory(mapping_context, dir_path, origin, mode = None, user = None, g
group = group or mapping_context.default_group,
uid = uid or mapping_context.default_uid,
gid = gid or mapping_context.default_gid,
)
))

def add_empty_file(mapping_context, dest_path, origin, mode = None, user = None, group = None, uid = None, gid = None):
"""Add a single file to the content map.
Expand All @@ -343,7 +354,7 @@ def add_empty_file(mapping_context, dest_path, origin, mode = None, user = None,
"""
dest = dest_path.strip("/")
_check_dest(mapping_context.content_map, dest, None, origin)
mapping_context.content_map[dest] = _DestFile(
_add_entry(mapping_context.content_map, dest, _DestFile(
src = None,
entry_type = ENTRY_IS_EMPTY_FILE,
origin = origin,
Expand All @@ -352,7 +363,7 @@ def add_empty_file(mapping_context, dest_path, origin, mode = None, user = None,
group = group or mapping_context.default_group,
uid = uid or mapping_context.default_uid,
gid = gid or mapping_context.default_gid,
)
))

def add_label_list(mapping_context, srcs):
"""Helper method to add a list of labels (typically 'srcs') to a content_map.
Expand Down Expand Up @@ -463,7 +474,7 @@ def add_from_default_info(
else:
entry_type = ENTRY_IS_FILE

mapping_context.content_map[d_path] = _DestFile(
_add_entry(mapping_context.content_map, d_path, _DestFile(
src = rf,
entry_type = entry_type,
origin = src.label,
Expand All @@ -472,7 +483,7 @@ def add_from_default_info(
group = mapping_context.default_group,
uid = mapping_context.default_uid,
gid = mapping_context.default_gid,
)
))

if runfiles.files:
mapping_context.file_deps_transitive.append(runfiles.files)
Expand Down Expand Up @@ -572,7 +583,7 @@ def add_single_file(mapping_context, dest_path, src, origin, mode = None, user =
else:
entry_type = ENTRY_IS_FILE

mapping_context.content_map[dest] = _DestFile(
_add_entry(mapping_context.content_map, dest, _DestFile(
src = src,
entry_type = entry_type,
origin = origin,
Expand All @@ -581,7 +592,7 @@ def add_single_file(mapping_context, dest_path, src, origin, mode = None, user =
group = group or mapping_context.default_group,
uid = uid or mapping_context.default_uid,
gid = gid or mapping_context.default_gid,
)
))

def add_symlink(mapping_context, dest_path, src, origin):
"""Add a symlink to the content map.
Expand All @@ -598,7 +609,7 @@ def add_symlink(mapping_context, dest_path, src, origin):
"""
dest = dest_path.strip("/")
_check_dest(mapping_context.content_map, dest, None, origin)
mapping_context.content_map[dest] = _DestFile(
_add_entry(mapping_context.content_map, dest, _DestFile(
src = None,
link_to = src,
entry_type = ENTRY_IS_LINK,
Expand All @@ -608,7 +619,7 @@ def add_symlink(mapping_context, dest_path, src, origin):
group = mapping_context.default_group,
uid = mapping_context.default_uid,
gid = mapping_context.default_gid,
)
))

def add_tree_artifact(content_map, dest_path, src, origin, mode = None, user = None, group = None, uid = None, gid = None):
"""Add an tree artifact (directory output) to the content map.
Expand All @@ -624,7 +635,7 @@ def add_tree_artifact(content_map, dest_path, src, origin, mode = None, user = N
uid: User id for the entry (probably unused)
gid: Group id for the entry (probably unused)
"""
content_map[dest_path] = _DestFile(
_add_entry(content_map, dest_path, _DestFile(
src = src,
origin = origin,
entry_type = ENTRY_IS_TREE,
Expand All @@ -633,7 +644,7 @@ def add_tree_artifact(content_map, dest_path, src, origin, mode = None, user = N
group = group,
uid = uid,
gid = gid,
)
))

def write_manifest(ctx, manifest_file, content_map, use_short_path = False, pretty_print = False):
"""Write a content map to a manifest file.
Expand All @@ -655,8 +666,9 @@ def write_manifest(ctx, manifest_file, content_map, use_short_path = False, pret
manifest_file,
"[\n" + ",\n".join(
[
_encode_manifest_entry(ctx, dst, content_map[dst], use_short_path, pretty_print)
_encode_manifest_entry(ctx, dst, entry, use_short_path, pretty_print)
for dst in sorted(content_map.keys())
for entry in content_map[dst]
],
) + "\n]\n",
)
Expand Down
4 changes: 3 additions & 1 deletion pkg/private/tar/build_tar.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ def __init__(self, output, directory, compression, compressor, create_parents,
self.preserve_mtime = preserve_mtime

def __enter__(self):
self.tree_files = manifest.TreeFileTracker()
self.tarfile = tar_writer.TarFileWriter(
self.output,
self.compression,
Expand Down Expand Up @@ -284,6 +285,7 @@ def add_tree(self, tree_top, destpath, mode=None, ids=None, names=None):
else:
dest_dir = dest
for dir in dirs:
self.tree_files.add_dir(dest_dir + dir, tree_top)
to_write[dest_dir + dir] = None
for file in sorted(files):
content_path = os.path.abspath(os.path.join(root, file))
Expand All @@ -308,7 +310,7 @@ def add_tree(self, tree_top, destpath, mode=None, ids=None, names=None):
ids=ids,
names=names,
kind=tarfile.DIRTYPE)
else:
elif self.tree_files.add_file(path, content_path, tree_top):
# If mode is unspecified, derive the mode from the file's mode.
if mode is None:
f_mode = 0o755 if os.access(content_path, os.X_OK) else 0o644
Expand Down
16 changes: 14 additions & 2 deletions pkg/private/zip/build_zip.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ def __init__(self, output_path: str, time_stamp: int, default_mode: int, compres
self.output_path = output_path
self.time_stamp = time_stamp
self.default_mode = default_mode
self.tree_files = manifest.TreeFileTracker()
compressions = {
"deflated": zipfile.ZIP_DEFLATED,
"lzma": zipfile.ZIP_LZMA,
Expand Down Expand Up @@ -224,6 +225,7 @@ def add_tree(self, tree_top: str, destpath: str, mode: int):
dest_dir = dest + rel_path_from_top + '/'
else:
dest_dir = dest
self.tree_files.add_dir(dest_dir, tree_top)
to_write[dest_dir] = None
for file in files:
content_path = os.path.abspath(os.path.join(root, file))
Expand All @@ -239,6 +241,8 @@ def add_tree(self, tree_top: str, destpath: str, mode: int):
for path in sorted(to_write.keys()):
content_path = to_write[path]
if content_path:
if not self.tree_files.add_file(path, content_path, tree_top):
continue
# If mode is unspecified, derive the mode from the file's mode.
if mode is None:
f_mode = "0o755" if os.access(content_path, os.X_OK) else self.default_mode
Expand All @@ -253,6 +257,8 @@ def add_tree(self, tree_top: str, destpath: str, mode: int):
dir_path = path
if not dir_path.endswith('/'):
dir_path += '/'
if dir_path in self.zip_file.NameToInfo:
continue
entry_info = self.make_zipinfo(path=dir_path, mode="0o755")
entry_info.compress_type = zipfile.ZIP_STORED
# Set directory bits
Expand All @@ -261,10 +267,16 @@ def add_tree(self, tree_top: str, destpath: str, mode: int):

def _load_manifest(prefix, manifest_path):
manifest_map = {}
shared_dest_trees = []

for entry in manifest.read_entries_from(manifest_path):
entry.dest = _combine_paths(prefix, entry.dest)
manifest_map[entry.dest] = entry
existing = manifest_map.get(entry.dest)
if (entry.type == manifest.ENTRY_IS_TREE and existing is not None and
existing.type == manifest.ENTRY_IS_TREE):
shared_dest_trees.append(entry)
else:
manifest_map[entry.dest] = entry

# We modify the dictionary as we're iterating over it, so we need to listify
# the keys here.
Expand All @@ -288,7 +300,7 @@ def _load_manifest(prefix, manifest_path):
origin = "parent directory of {}".format(manifest_map[dest].origin),
)

return sorted(manifest_map.values(), key = lambda x: x.dest)
return sorted(list(manifest_map.values()) + shared_dest_trees, key = lambda x: x.dest)

def main(args):
unix_ts = max(ZIP_EPOCH, args.timestamp)
Expand Down
Loading
Loading