Skip to content

gh-89727: Fix os.walk to handle late editing of dirs #100703

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jonburdo wants to merge 14 commits into from
Closed

gh-89727: Fix os.walk to handle late editing of dirs #100703

Changes from 1 commit
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
2767f70
adopt old os.walk behavior to handle changes to dirs list after itera…
jonburdo Jan 3, 2023
04a16a2
move stack logic to bottom of loop in os.walk
jonburdo Jan 3, 2023
6b0465f
move scandir logic to bottom of loop
jonburdo Jan 3, 2023
2ff800f
use for-loop to process dirs in bottom-up clause
jonburdo Jan 3, 2023
072bdfb
Merge branch 'main' into os-walk-dirs-editing
jonburdo Jan 3, 2023
41d7463
return if scandir_it fails on first value
jonburdo Jan 3, 2023
d81ff45
add test for late modification of dirs in os.walk
jonburdo Jan 3, 2023
431d47a
Merge branch 'main' into os-walk-dirs-editing
jonburdo Jan 3, 2023
2829a7d
use while-else instead of while True, make variable names clearer
jonburdo Jan 3, 2023
b59358b
change os.walk docstring to say "traverse" instead of "recurse"
jonburdo Jan 3, 2023
839f31e
fix scandir comment
jonburdo Jan 4, 2023
1011e19
Merge branch 'main' into os-walk-dirs-editing
jonburdo Jan 4, 2023
80b7e82
split topdown and bottomup logic into separate functions for os.walk
jonburdo Jan 4, 2023
70cb0c2
Merge branch 'main' into os-walk-dirs-editing
erlend-aasland Jan 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
add test for late modification of dirs in os.walk
  • Loading branch information
@jonburdo
jonburdo committed Jan 3, 2023
commit d81ff45aaafb43900dd09b9d6f180b77ea0c0780
57 changes: 57 additions & 0 deletions Lib/test/test_os.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1512,6 +1512,57 @@ def test_walk_above_recursion_limit(self):
self.assertEqual(sorted(dirs), ["SUB1", "SUB2", "d"])
self.assertEqual(all, expected)

def test_walk_late_dirs_modification(self):
extra_parts = [
("SUB3", "SUB31"),
("SUB4", "SUB41"),
("SUB5", "SUB51"),
]
for dir_parts in extra_parts:
os.makedirs(os.path.join(self.walk_path, *dir_parts))

all = self.walk(self.walk_path)
result = []
result.append(next(all))
dirs = result[0][1]
dirs.sort()

# SUB1
# SUB1/SUB11
result.append(next(all))
result.append(next(all))
# SUB2 is removed
dirs.pop(1)
# swap SUB3 and SUB4
dirs[1], dirs[2] = dirs[2], dirs[1]
# SUB4
# SUB4/SUB41
# SUB3
result.append(next(all))
result.append(next(all))
result.append(next(all))
# clear while between SUB3 and SUB3/SUB31
dirs.clear()
result.extend(all)

expected_parts = [
([], [], ["tmp1"]),
(["SUB1"], ["SUB11"], ["tmp2"]),
(["SUB1", "SUB11"], [], []),
# SUB2 skipped
(["SUB4"], ["SUB41"], []),
(["SUB4", "SUB41"], [], []),
(["SUB3"], ["SUB31"], []),
(["SUB3", "SUB31"], [], []),
# SUB5 skipped
# SUB5/SUB51 skipped
]
expected = [
(os.path.join(self.walk_path, *r), d, f)
for r, d, f in expected_parts
]
self.assertEqual(result, expected)


@unittest.skipUnless(hasattr(os, 'fwalk'), "Test needs os.fwalk()")
class FwalkTests(WalkTests):
Expand Down Expand Up @@ -1603,6 +1654,10 @@ def walk(self, top, **kwargs):
bdirs[:] = list(map(os.fsencode, dirs))
bfiles[:] = list(map(os.fsencode, files))

# modifying bdirs in self.walk above prevents late modification
test_walk_late_dirs_modification = None


@unittest.skipUnless(hasattr(os, 'fwalk'), "Test needs os.fwalk()")
class BytesFwalkTests(FwalkTests):
"""Tests for os.walk() with bytes."""
Expand All @@ -1615,6 +1670,8 @@ def fwalk(self, top='.', *args, **kwargs):
bdirs[:] = list(map(os.fsencode, dirs))
bfiles[:] = list(map(os.fsencode, files))

test_walk_late_dirs_modification = None


class MakedirTests(unittest.TestCase):
def setUp(self):
Expand Down