Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions newsfragments/+pattern-normcase-cache.bugfix.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Reusing a ``matchers.Pattern`` across path modules now normalizes its pattern
for each module, avoiding order-dependent matching results.
58 changes: 53 additions & 5 deletions path/matchers.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,60 @@

import fnmatch
import ntpath
import types
from typing import TYPE_CHECKING, Any, Callable, overload

if TYPE_CHECKING:
from typing_extensions import Literal


def _same_normalizer(left: Callable, right: Callable) -> bool:
"""Compare callable normalizers without invoking custom equality."""
if left is right:
return True

if isinstance(left, types.MethodType) and isinstance(right, types.MethodType):
return left.__self__ is right.__self__ and left.__func__ is right.__func__

if isinstance(left, types.BuiltinMethodType) and isinstance(
right, types.BuiltinMethodType
):
# Built-in bound methods lack __func__, but their method objects are
# recreated on access and share their owner and method name.
owner = left.__self__
return (
owner is not None
and not isinstance(owner, types.ModuleType)
and owner is right.__self__
and left.__name__ == right.__name__
)

return False


class _PatternCache:
"""Transient cache state; omit callables when a Pattern is pickled."""

def __init__(self) -> None:
self._entry: tuple[Callable[[str], str], str] | None = None

def __reduce__(self) -> Any:
# Pattern's regular dict/slot state remains intact, while a possibly
# unpicklable normalizer is discarded and recomputed after loading.
return (type(self), ())

def get_pattern(self, pattern: str, normcase: Callable[[str], str]) -> str:
entry = self._entry
if entry is not None:
cached_normalizer, cached_pattern = entry
if _same_normalizer(cached_normalizer, normcase):
return cached_pattern

normalized = normcase(pattern)
self._entry = (normcase, normalized)
return normalized


@overload
def load(param: None) -> Null: ...

Expand Down Expand Up @@ -45,18 +93,18 @@ def __call__(self, path: str) -> Literal[True]:

class Pattern(Base):
pattern: str
_pattern: str
_pattern_cache: _PatternCache

def __init__(self, pattern: str):
self.pattern = pattern

def get_pattern(self, normcase: Callable[[str], str]) -> str:
try:
return self._pattern
cache = self._pattern_cache
except AttributeError:
pass
self._pattern = normcase(self.pattern)
return self._pattern
# Lazily initialize so older pickles without the cache remain valid.
cache = self._pattern_cache = _PatternCache()
return cache.get_pattern(self.pattern, normcase)

# NOTE: 'path' should be annotated with Path, but cannot due to circular imports.
def __call__(self, path) -> bool:
Expand Down
74 changes: 74 additions & 0 deletions tests/test_path.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import importlib
import ntpath
import os
import pickle
import platform
import posixpath
import re
Expand All @@ -36,6 +37,19 @@
from path import Multi, Path, SpecialResolver, TempDir, matchers


def _lowercase_normalizer():
return lambda self, value: value.lower()


class _LambdaPattern(matchers.Pattern):
__slots__ = ('label',)
normcase = _lowercase_normalizer()

def __init__(self, pattern, label):
super().__init__(pattern)
self.label = label


def os_choose(**choices):
"""Choose a value from several possible values, based on os.name"""
return choices[os.name]
Expand Down Expand Up @@ -1149,6 +1163,66 @@ def test_iterdir_custom_module(self, tmpdir):
assert list(p.iterdir('S*')) == [p / 'sub']
assert list(p.iterdir('f*')) == [p / 'File']

def test_reused_pattern_respects_path_module(self, tmpdir):
root = Path(tmpdir)
(root / 'File.TXT').touch()
always_unix = Path.using_module(posixpath)
always_win = Path.using_module(ntpath)
paths = (always_unix(tmpdir), always_win(tmpdir))

for first, second in (paths, paths[::-1]):
pattern = matchers.Pattern('*.TXT')
assert [path.name for path in first.iterdir(pattern)] == ['File.TXT']
assert [path.name for path in second.iterdir(pattern)] == ['File.TXT']

def test_reused_bound_normalizer_uses_cache(self):
class Normalizer:
calls = 0

def normcase(self, value):
self.calls += 1
return value.lower()

normalizer = Normalizer()
pattern = matchers.Pattern('*.TXT')
assert pattern.get_pattern(normalizer.normcase) == '*.txt'
assert pattern.get_pattern(normalizer.normcase) == '*.txt'
assert normalizer.calls == 1

def test_reused_builtin_bound_normalizer_uses_cache(self):
normalizer = {'*.TXT': '*.txt'}
pattern = matchers.Pattern('*.TXT')

assert pattern.get_pattern(normalizer.get) == '*.txt'
assert pattern.get_pattern(normalizer.get) == '*.txt'

def test_distinct_callable_normalizers_do_not_alias_by_metadata(self):
class Normalizer:
__self__ = object()
__name__ = 'normcase'

def __init__(self, normalize):
self.normalize = normalize

def __call__(self, value):
return self.normalize(value)

pattern = matchers.Pattern('*.TXT')
lowercase = Normalizer(str.lower)
identity = Normalizer(lambda value: value)

assert pattern.get_pattern(lowercase) == '*.txt'
assert pattern.get_pattern(identity) == '*.TXT'

def test_lambda_normalizer_pattern_round_trips_after_matching(self):
pattern = _LambdaPattern('*.TXT', 'custom state')
assert pattern(Path('File.TXT'))

restored = pickle.loads(pickle.dumps(pattern))

assert restored.label == 'custom state'
assert restored(Path('File.TXT'))

def test_iterdir_case_insensitive(self, tmpdir):
"""
Listdir patterns should honor the case sensitivity of the path module
Expand Down