Skip to content

gh-89727: Fix os.walk to handle late editing of dirs #100703

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jonburdo wants to merge 14 commits into from
Closed

gh-89727: Fix os.walk to handle late editing of dirs #100703

Changes from 1 commit
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
2767f70
adopt old os.walk behavior to handle changes to dirs list after itera…
jonburdo Jan 3, 2023
04a16a2
move stack logic to bottom of loop in os.walk
jonburdo Jan 3, 2023
6b0465f
move scandir logic to bottom of loop
jonburdo Jan 3, 2023
2ff800f
use for-loop to process dirs in bottom-up clause
jonburdo Jan 3, 2023
072bdfb
Merge branch 'main' into os-walk-dirs-editing
jonburdo Jan 3, 2023
41d7463
return if scandir_it fails on first value
jonburdo Jan 3, 2023
d81ff45
add test for late modification of dirs in os.walk
jonburdo Jan 3, 2023
431d47a
Merge branch 'main' into os-walk-dirs-editing
jonburdo Jan 3, 2023
2829a7d
use while-else instead of while True, make variable names clearer
jonburdo Jan 3, 2023
b59358b
change os.walk docstring to say "traverse" instead of "recurse"
jonburdo Jan 3, 2023
839f31e
fix scandir comment
jonburdo Jan 4, 2023
1011e19
Merge branch 'main' into os-walk-dirs-editing
jonburdo Jan 4, 2023
80b7e82
split topdown and bottomup logic into separate functions for os.walk
jonburdo Jan 4, 2023
70cb0c2
Merge branch 'main' into os-walk-dirs-editing
erlend-aasland Jan 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
split topdown and bottomup logic into separate functions for os.walk
  • Loading branch information
@jonburdo
jonburdo committed Jan 4, 2023
commit 80b7e825bced8e9bd256d6f154822695fc363f75
161 changes: 96 additions & 65 deletions Lib/os.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -340,14 +340,12 @@ def walk(top, topdown=True, onerror=None, followlinks=False):

"""
sys.audit("os.walk", top, topdown, onerror, followlinks)

top = fspath(top)
if topdown:
return _walk_topdown(top, onerror, followlinks)
return _walk_bottomup(top, onerror, followlinks)

# We may not have read permission for top, in which case we can't
# get a list of the files the directory contains.
# We suppress the exception here, rather than blow up for a
# minor reason when (say) a thousand readable directories are still
# left to visit.
def _walk_topdown(top, onerror, followlinks):
try:
scandir_it = scandir(top)
except OSError as error:
Expand All @@ -360,16 +358,13 @@ def walk(top, topdown=True, onerror=None, followlinks=False):
while True:
dirs = []
nondirs = []
walk_dirs = []
use_entry = True

with scandir_it:
while True:
try:
try:
entry = next(scandir_it)
except StopIteration:
break
entry = next(scandir_it)
except StopIteration:
break
except OSError as error:
if onerror is not None:
onerror(error)
Expand All @@ -388,7 +383,72 @@ def walk(top, topdown=True, onerror=None, followlinks=False):
else:
nondirs.append(entry.name)

if not topdown and is_dir:
if use_entry:
# Yield before sub-directory traversal if going top down
yield top, dirs, nondirs
# Append dir names to walk along with top, their parent dir
stack.append([top, iter(dirs)])

# Set top and scandir_it for the next iteration
while stack:
root, dirname_iter = stack[-1]
for dirname in dirname_iter:
top = join(root, dirname)
# bpo-23605: os.path.islink() is used instead of caching
# entry.is_symlink() result during the loop on os.scandir() because
# the caller can replace the directory entry during the previous
# "yield"
if followlinks or not islink(top):
try:
scandir_it = scandir(top)
except OSError as error:
if onerror is not None:
onerror(error)
else:
break
else:
stack.pop()
continue
break
else:
return

def _walk_bottomup(top, onerror, followlinks):
try:
scandir_it = scandir(top)
except OSError as error:
if onerror is not None:
onerror(error)
return

stack = []
islink, join = path.islink, path.join
while True:
dirs = []
nondirs = []
walk_dirs = []
use_entry = True
with scandir_it:
while True:
try:
entry = next(scandir_it)
except StopIteration:
break
except OSError as error:
if onerror is not None:
onerror(error)
use_entry = False
break

try:
is_dir = entry.is_dir()
except OSError:
# If is_dir() raises an OSError, consider the entry not to
# be a directory, same behaviour as os.path.isdir().
is_dir = False

if is_dir:
dirs.append(entry.name)
# Bottom-up: traverse into sub-directory, but exclude
# symlinks to directories if followlinks is False
if followlinks:
Expand All @@ -405,61 +465,32 @@ def walk(top, topdown=True, onerror=None, followlinks=False):

if walk_into:
walk_dirs.append(entry.path)
if topdown:
if use_entry:
# Yield before sub-directory traversal if going top down
yield top, dirs, nondirs
# Append dir names to walk along with top, their parent dir
stack.append([top, iter(dirs)])

# Set top and scandir_it for the next iteration
while stack:
root, dirs = stack[-1]
for dirname in dirs:
top = join(root, dirname)
# bpo-23605: os.path.islink() is used instead of caching
# entry.is_symlink() result during the loop on os.scandir() because
# the caller can replace the directory entry during the previous
# "yield"
if followlinks or not islink(top):
try:
scandir_it = scandir(top)
except OSError as error:
if onerror is not None:
onerror(error)
else:
break
else:
stack.pop()
continue
break
else:
return
else:
if use_entry:
# Traverse into sub-directories by appending a value
# (entry, dirs) where dirs will be walked and then
# when dirs is exhausted entry will be yielded
stack.append(((top, dirs, nondirs), iter(walk_dirs)))

# Set top and scandir_it for the next iteration
while stack:
entry, dirs = stack[-1]
for top in dirs:
try:
scandir_it = scandir(top)
except OSError as error:
if onerror is not None:
onerror(error)
else:
break
nondirs.append(entry.name)
if use_entry:
# Traverse into sub-directories by appending a value
# (entry, dirs) where dirs will be walked and then
# when dirs is exhausted entry will be yielded
stack.append(((top, dirs, nondirs), iter(walk_dirs)))

# Set top and scandir_it for the next iteration
while stack:
entry, dirpath_iter = stack[-1]
for top in dirpath_iter:
try:
scandir_it = scandir(top)
except OSError as error:
if onerror is not None:
onerror(error)
else:
stack.pop()
yield entry
continue
break
break
else:
return
stack.pop()
yield entry
continue
break
else:
return

__all__.append("walk")

Expand Down