diff --git a/.gitignore b/.gitignore index 695bdda2..46b1ae3e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,18 @@ .idea/ .DS_Store __pycache__ -./Code 2. Cartpole/6. A3C/Cartpole_A3C.pgy \ No newline at end of file +.venv/ +*.egg-info/ +.python-version +*.pt +wandb/ +logs/ +./Code 2. Cartpole/6. A3C/Cartpole_A3C.pgy +# Local scratch scripts +scripts/ + +# Local-only docs (not for github) +docs/ + +# Local-only collaboration principles for Claude +CLAUDE.md diff --git a/1-grid-world/1-policy-iteration/environment.py b/1-grid-world/1-policy-iteration/environment.py deleted file mode 100644 index 910d4ba8..00000000 --- a/1-grid-world/1-policy-iteration/environment.py +++ /dev/null @@ -1,245 +0,0 @@ -import tkinter as tk -from tkinter import Button -import time -import numpy as np -from PIL import ImageTk, Image - -PhotoImage = ImageTk.PhotoImage -UNIT = 100 # pixels -HEIGHT = 5 # grid height -WIDTH = 5 # grid width -TRANSITION_PROB = 1 -POSSIBLE_ACTIONS = [0, 1, 2, 3] # up, down, left, right -ACTIONS = [(-1, 0), (1, 0), (0, -1), (0, 1)] # actions in coordinates -REWARDS = [] - - -class GraphicDisplay(tk.Tk): - def __init__(self, agent): - super(GraphicDisplay, self).__init__() - self.title('Policy Iteration') - self.geometry('{0}x{1}'.format(HEIGHT * UNIT, HEIGHT * UNIT + 50)) - self.texts = [] - self.arrows = [] - self.env = Env() - self.agent = agent - self.evaluation_count = 0 - self.improvement_count = 0 - self.is_moving = 0 - (self.up, self.down, self.left, self.right), self.shapes = self.load_images() - self.canvas = self._build_canvas() - self.text_reward(2, 2, "R : 1.0") - self.text_reward(1, 2, "R : -1.0") - self.text_reward(2, 1, "R : -1.0") - - def _build_canvas(self): - canvas = tk.Canvas(self, bg='white', - height=HEIGHT * UNIT, - width=WIDTH * UNIT) - # buttons - iteration_button = Button(self, text="Evaluate", - command=self.evaluate_policy) - iteration_button.configure(width=10, activebackground="#33B5E5") - canvas.create_window(WIDTH * UNIT * 0.13, HEIGHT * UNIT + 10, - window=iteration_button) - policy_button = Button(self, text="Improve", - command=self.improve_policy) - policy_button.configure(width=10, activebackground="#33B5E5") - canvas.create_window(WIDTH * UNIT * 0.37, HEIGHT * UNIT + 10, - window=policy_button) - policy_button = Button(self, text="move", command=self.move_by_policy) - policy_button.configure(width=10, activebackground="#33B5E5") - canvas.create_window(WIDTH * UNIT * 0.62, HEIGHT * UNIT + 10, - window=policy_button) - policy_button = Button(self, text="reset", command=self.reset) - policy_button.configure(width=10, activebackground="#33B5E5") - canvas.create_window(WIDTH * UNIT * 0.87, HEIGHT * UNIT + 10, - window=policy_button) - - # create grids - for col in range(0, WIDTH * UNIT, UNIT): # 0~400 by 80 - x0, y0, x1, y1 = col, 0, col, HEIGHT * UNIT - canvas.create_line(x0, y0, x1, y1) - for row in range(0, HEIGHT * UNIT, UNIT): # 0~400 by 80 - x0, y0, x1, y1 = 0, row, HEIGHT * UNIT, row - canvas.create_line(x0, y0, x1, y1) - - # add img to canvas - self.rectangle = canvas.create_image(50, 50, image=self.shapes[0]) - canvas.create_image(250, 150, image=self.shapes[1]) - canvas.create_image(150, 250, image=self.shapes[1]) - canvas.create_image(250, 250, image=self.shapes[2]) - - # pack all - canvas.pack() - - return canvas - - def load_images(self): - up = PhotoImage(Image.open("../img/up.png").resize((13, 13))) - right = PhotoImage(Image.open("../img/right.png").resize((13, 13))) - left = PhotoImage(Image.open("../img/left.png").resize((13, 13))) - down = PhotoImage(Image.open("../img/down.png").resize((13, 13))) - rectangle = PhotoImage(Image.open("../img/rectangle.png").resize((65, 65))) - triangle = PhotoImage(Image.open("../img/triangle.png").resize((65, 65))) - circle = PhotoImage(Image.open("../img/circle.png").resize((65, 65))) - return (up, down, left, right), (rectangle, triangle, circle) - - def reset(self): - if self.is_moving == 0: - self.evaluation_count = 0 - self.improvement_count = 0 - for i in self.texts: - self.canvas.delete(i) - - for i in self.arrows: - self.canvas.delete(i) - self.agent.value_table = [[0.0] * WIDTH for _ in range(HEIGHT)] - self.agent.policy_table = ([[[0.25, 0.25, 0.25, 0.25]] * WIDTH - for _ in range(HEIGHT)]) - self.agent.policy_table[2][2] = [] - x, y = self.canvas.coords(self.rectangle) - self.canvas.move(self.rectangle, UNIT / 2 - x, UNIT / 2 - y) - - def text_value(self, row, col, contents, font='Helvetica', size=10, - style='normal', anchor="nw"): - origin_x, origin_y = 85, 70 - x, y = origin_y + (UNIT * col), origin_x + (UNIT * row) - font = (font, str(size), style) - text = self.canvas.create_text(x, y, fill="black", text=contents, - font=font, anchor=anchor) - return self.texts.append(text) - - def text_reward(self, row, col, contents, font='Helvetica', size=10, - style='normal', anchor="nw"): - origin_x, origin_y = 5, 5 - x, y = origin_y + (UNIT * col), origin_x + (UNIT * row) - font = (font, str(size), style) - text = self.canvas.create_text(x, y, fill="black", text=contents, - font=font, anchor=anchor) - return self.texts.append(text) - - def rectangle_move(self, action): - base_action = np.array([0, 0]) - location = self.find_rectangle() - self.render() - if action == 0 and location[0] > 0: # up - base_action[1] -= UNIT - elif action == 1 and location[0] < HEIGHT - 1: # down - base_action[1] += UNIT - elif action == 2 and location[1] > 0: # left - base_action[0] -= UNIT - elif action == 3 and location[1] < WIDTH - 1: # right - base_action[0] += UNIT - # move agent - self.canvas.move(self.rectangle, base_action[0], base_action[1]) - - def find_rectangle(self): - temp = self.canvas.coords(self.rectangle) - x = (temp[0] / 100) - 0.5 - y = (temp[1] / 100) - 0.5 - return int(y), int(x) - - def move_by_policy(self): - if self.improvement_count != 0 and self.is_moving != 1: - self.is_moving = 1 - - x, y = self.canvas.coords(self.rectangle) - self.canvas.move(self.rectangle, UNIT / 2 - x, UNIT / 2 - y) - - x, y = self.find_rectangle() - while len(self.agent.policy_table[x][y]) != 0: - self.after(100, - self.rectangle_move(self.agent.get_action([x, y]))) - x, y = self.find_rectangle() - self.is_moving = 0 - - def draw_one_arrow(self, col, row, policy): - if col == 2 and row == 2: - return - - if policy[0] > 0: # up - origin_x, origin_y = 50 + (UNIT * row), 10 + (UNIT * col) - self.arrows.append(self.canvas.create_image(origin_x, origin_y, - image=self.up)) - if policy[1] > 0: # down - origin_x, origin_y = 50 + (UNIT * row), 90 + (UNIT * col) - self.arrows.append(self.canvas.create_image(origin_x, origin_y, - image=self.down)) - if policy[2] > 0: # left - origin_x, origin_y = 10 + (UNIT * row), 50 + (UNIT * col) - self.arrows.append(self.canvas.create_image(origin_x, origin_y, - image=self.left)) - if policy[3] > 0: # right - origin_x, origin_y = 90 + (UNIT * row), 50 + (UNIT * col) - self.arrows.append(self.canvas.create_image(origin_x, origin_y, - image=self.right)) - - def draw_from_policy(self, policy_table): - for i in range(HEIGHT): - for j in range(WIDTH): - self.draw_one_arrow(i, j, policy_table[i][j]) - - def print_value_table(self, value_table): - for i in range(WIDTH): - for j in range(HEIGHT): - self.text_value(i, j, value_table[i][j]) - - def render(self): - time.sleep(0.1) - self.canvas.tag_raise(self.rectangle) - self.update() - - def evaluate_policy(self): - self.evaluation_count += 1 - for i in self.texts: - self.canvas.delete(i) - self.agent.policy_evaluation() - self.print_value_table(self.agent.value_table) - - def improve_policy(self): - self.improvement_count += 1 - for i in self.arrows: - self.canvas.delete(i) - self.agent.policy_improvement() - self.draw_from_policy(self.agent.policy_table) - - -class Env: - def __init__(self): - self.transition_probability = TRANSITION_PROB - self.width = WIDTH - self.height = HEIGHT - self.reward = [[0] * WIDTH for _ in range(HEIGHT)] - self.possible_actions = POSSIBLE_ACTIONS - self.reward[2][2] = 1 # reward 1 for circle - self.reward[1][2] = -1 # reward -1 for triangle - self.reward[2][1] = -1 # reward -1 for triangle - self.all_state = [] - - for x in range(WIDTH): - for y in range(HEIGHT): - state = [x, y] - self.all_state.append(state) - - def get_reward(self, state, action): - next_state = self.state_after_action(state, action) - return self.reward[next_state[0]][next_state[1]] - - def state_after_action(self, state, action_index): - action = ACTIONS[action_index] - return self.check_boundary([state[0] + action[0], state[1] + action[1]]) - - @staticmethod - def check_boundary(state): - state[0] = (0 if state[0] < 0 else WIDTH - 1 - if state[0] > WIDTH - 1 else state[0]) - state[1] = (0 if state[1] < 0 else HEIGHT - 1 - if state[1] > HEIGHT - 1 else state[1]) - return state - - def get_transition_prob(self, state, action): - return self.transition_probability - - def get_all_states(self): - return self.all_state diff --git a/1-grid-world/1-policy-iteration/policy_iteration.py b/1-grid-world/1-policy_iteration.py similarity index 77% rename from 1-grid-world/1-policy-iteration/policy_iteration.py rename to 1-grid-world/1-policy_iteration.py index d6dc414e..7dc85ee2 100644 --- a/1-grid-world/1-policy-iteration/policy_iteration.py +++ b/1-grid-world/1-policy_iteration.py @@ -1,6 +1,6 @@ -# -*- coding: utf-8 -*- import random -from environment import GraphicDisplay, Env + +from env import GraphicDisplay, PolicyEnv as Env class PolicyIteration: @@ -98,5 +98,31 @@ def get_value(self, state): if __name__ == "__main__": env = Env() policy_iteration = PolicyIteration(env) - grid_world = GraphicDisplay(policy_iteration) - grid_world.mainloop() + display = GraphicDisplay(policy_iteration, title="Policy Iteration") + + def on_evaluate(): + policy_iteration.policy_evaluation() + display.show_values(policy_iteration.value_table) + + def on_improve(): + policy_iteration.policy_improvement() + display.show_arrows(policy_iteration.policy_table) + + def on_move(): + display.move_along_policy(policy_iteration.get_action) + + def on_reset(): + policy_iteration.__init__(env) + display.clear() + display.agent_pos = [0, 0] + display.clicks.clear() + + # Workflow: (Evaluate x several -> Improve) x several -> Move. + # Improve unlocks after Evaluate; Move unlocks after Improve. + display.buttons = [ + ("Evaluate", on_evaluate), + ("Improve", on_improve, lambda: display.click_count("Evaluate") > 0), + ("Move", on_move, lambda: display.click_count("Improve") > 0), + ("Reset", on_reset), + ] + display.mainloop() diff --git a/1-grid-world/2-value-iteration/environment.py b/1-grid-world/2-value-iteration/environment.py deleted file mode 100644 index 81af3dc5..00000000 --- a/1-grid-world/2-value-iteration/environment.py +++ /dev/null @@ -1,261 +0,0 @@ -import tkinter as tk -import time -import numpy as np -import random -from PIL import ImageTk, Image - -PhotoImage = ImageTk.PhotoImage -UNIT = 100 # pixels -HEIGHT = 5 # grid height -WIDTH = 5 # grid width -TRANSITION_PROB = 1 -POSSIBLE_ACTIONS = [0, 1, 2, 3] # up, down, left, right -ACTIONS = [(-1, 0), (1, 0), (0, -1), (0, 1)] # actions in coordinates -REWARDS = [] - - -class GraphicDisplay(tk.Tk): - def __init__(self, value_iteration): - super(GraphicDisplay, self).__init__() - self.title('Value Iteration') - self.geometry('{0}x{1}'.format(HEIGHT * UNIT, HEIGHT * UNIT + 50)) - self.texts = [] - self.arrows = [] - self.env = Env() - self.agent = value_iteration - self.iteration_count = 0 - self.improvement_count = 0 - self.is_moving = 0 - (self.up, self.down, self.left, - self.right), self.shapes = self.load_images() - self.canvas = self._build_canvas() - self.text_reward(2, 2, "R : 1.0") - self.text_reward(1, 2, "R : -1.0") - self.text_reward(2, 1, "R : -1.0") - - def _build_canvas(self): - canvas = tk.Canvas(self, bg='white', - height=HEIGHT * UNIT, - width=WIDTH * UNIT) - # buttons - iteration_button = tk.Button(self, text="Calculate", - command=self.calculate_value) - iteration_button.configure(width=10, activebackground="#33B5E5") - canvas.create_window(WIDTH * UNIT * 0.13, (HEIGHT * UNIT) + 10, - window=iteration_button) - - policy_button = tk.Button(self, text="Print Policy", - command=self.print_optimal_policy) - policy_button.configure(width=10, activebackground="#33B5E5") - canvas.create_window(WIDTH * UNIT * 0.37, (HEIGHT * UNIT) + 10, - window=policy_button) - - policy_button = tk.Button(self, text="Move", - command=self.move_by_policy) - policy_button.configure(width=10, activebackground="#33B5E5") - canvas.create_window(WIDTH * UNIT * 0.62, (HEIGHT * UNIT) + 10, - window=policy_button) - - policy_button = tk.Button(self, text="Clear", command=self.clear) - policy_button.configure(width=10, activebackground="#33B5E5") - canvas.create_window(WIDTH * UNIT * 0.87, (HEIGHT * UNIT) + 10, - window=policy_button) - - # create grids - for col in range(0, WIDTH * UNIT, UNIT): # 0~400 by 80 - x0, y0, x1, y1 = col, 0, col, HEIGHT * UNIT - canvas.create_line(x0, y0, x1, y1) - for row in range(0, HEIGHT * UNIT, UNIT): # 0~400 by 80 - x0, y0, x1, y1 = 0, row, HEIGHT * UNIT, row - canvas.create_line(x0, y0, x1, y1) - - # add img to canvas - self.rectangle = canvas.create_image(50, 50, image=self.shapes[0]) - canvas.create_image(250, 150, image=self.shapes[1]) - canvas.create_image(150, 250, image=self.shapes[1]) - canvas.create_image(250, 250, image=self.shapes[2]) - - # pack all - canvas.pack() - - return canvas - - def load_images(self): - PhotoImage = ImageTk.PhotoImage - up = PhotoImage(Image.open("../img/up.png").resize((13, 13))) - right = PhotoImage(Image.open("../img/right.png").resize((13, 13))) - left = PhotoImage(Image.open("../img/left.png").resize((13, 13))) - down = PhotoImage(Image.open("../img/down.png").resize((13, 13))) - rectangle = PhotoImage( - Image.open("../img/rectangle.png").resize((65, 65))) - triangle = PhotoImage( - Image.open("../img/triangle.png").resize((65, 65))) - circle = PhotoImage(Image.open("../img/circle.png").resize((65, 65))) - return (up, down, left, right), (rectangle, triangle, circle) - - def clear(self): - - if self.is_moving == 0: - self.iteration_count = 0 - self.improvement_count = 0 - for i in self.texts: - self.canvas.delete(i) - - for i in self.arrows: - self.canvas.delete(i) - - self.agent.value_table = [[0.0] * WIDTH for _ in range(HEIGHT)] - - x, y = self.canvas.coords(self.rectangle) - self.canvas.move(self.rectangle, UNIT / 2 - x, UNIT / 2 - y) - - def reset(self): - self.update() - time.sleep(0.5) - self.canvas.delete(self.rectangle) - return self.canvas.coords(self.rectangle) - - def text_value(self, row, col, contents, font='Helvetica', size=12, - style='normal', anchor="nw"): - origin_x, origin_y = 85, 70 - x, y = origin_y + (UNIT * col), origin_x + (UNIT * row) - font = (font, str(size), style) - text = self.canvas.create_text(x, y, fill="black", text=contents, - font=font, anchor=anchor) - return self.texts.append(text) - - def text_reward(self, row, col, contents, font='Helvetica', size=12, - style='normal', anchor="nw"): - origin_x, origin_y = 5, 5 - x, y = origin_y + (UNIT * col), origin_x + (UNIT * row) - font = (font, str(size), style) - text = self.canvas.create_text(x, y, fill="black", text=contents, - font=font, anchor=anchor) - return self.texts.append(text) - - def rectangle_move(self, action): - base_action = np.array([0, 0]) - location = self.find_rectangle() - self.render() - if action == 0 and location[0] > 0: # up - base_action[1] -= UNIT - elif action == 1 and location[0] < HEIGHT - 1: # down - base_action[1] += UNIT - elif action == 2 and location[1] > 0: # left - base_action[0] -= UNIT - elif action == 3 and location[1] < WIDTH - 1: # right - base_action[0] += UNIT - - self.canvas.move(self.rectangle, base_action[0], - base_action[1]) # move agent - - def find_rectangle(self): - temp = self.canvas.coords(self.rectangle) - x = (temp[0] / 100) - 0.5 - y = (temp[1] / 100) - 0.5 - return int(y), int(x) - - def move_by_policy(self): - - if self.improvement_count != 0 and self.is_moving != 1: - self.is_moving = 1 - x, y = self.canvas.coords(self.rectangle) - self.canvas.move(self.rectangle, UNIT / 2 - x, UNIT / 2 - y) - - x, y = self.find_rectangle() - while len(self.agent.get_action([x, y])) != 0: - action = random.sample(self.agent.get_action([x, y]), 1)[0] - self.after(100, self.rectangle_move(action)) - x, y = self.find_rectangle() - self.is_moving = 0 - - def draw_one_arrow(self, col, row, action): - if col == 2 and row == 2: - return - if action == 0: # up - origin_x, origin_y = 50 + (UNIT * row), 10 + (UNIT * col) - self.arrows.append(self.canvas.create_image(origin_x, origin_y, - image=self.up)) - elif action == 1: # down - origin_x, origin_y = 50 + (UNIT * row), 90 + (UNIT * col) - self.arrows.append(self.canvas.create_image(origin_x, origin_y, - image=self.down)) - elif action == 3: # right - origin_x, origin_y = 90 + (UNIT * row), 50 + (UNIT * col) - self.arrows.append(self.canvas.create_image(origin_x, origin_y, - image=self.right)) - elif action == 2: # left - origin_x, origin_y = 10 + (UNIT * row), 50 + (UNIT * col) - self.arrows.append(self.canvas.create_image(origin_x, origin_y, - image=self.left)) - - def draw_from_values(self, state, action_list): - i = state[0] - j = state[1] - for action in action_list: - self.draw_one_arrow(i, j, action) - - def print_values(self, values): - for i in range(WIDTH): - for j in range(HEIGHT): - self.text_value(i, j, values[i][j]) - - def render(self): - time.sleep(0.1) - self.canvas.tag_raise(self.rectangle) - self.update() - - def calculate_value(self): - self.iteration_count += 1 - for i in self.texts: - self.canvas.delete(i) - self.agent.value_iteration() - self.print_values(self.agent.value_table) - - def print_optimal_policy(self): - self.improvement_count += 1 - for i in self.arrows: - self.canvas.delete(i) - for state in self.env.get_all_states(): - action = self.agent.get_action(state) - self.draw_from_values(state, action) - - -class Env: - def __init__(self): - self.transition_probability = TRANSITION_PROB - self.width = WIDTH # Width of Grid World - self.height = HEIGHT # Height of GridWorld - self.reward = [[0] * WIDTH for _ in range(HEIGHT)] - self.possible_actions = POSSIBLE_ACTIONS - self.reward[2][2] = 1 # reward 1 for circle - self.reward[1][2] = -1 # reward -1 for triangle - self.reward[2][1] = -1 # reward -1 for triangle - self.all_state = [] - - for x in range(WIDTH): - for y in range(HEIGHT): - state = [x, y] - self.all_state.append(state) - - def get_reward(self, state, action): - next_state = self.state_after_action(state, action) - return self.reward[next_state[0]][next_state[1]] - - def state_after_action(self, state, action_index): - action = ACTIONS[action_index] - return self.check_boundary([state[0] + action[0], state[1] + action[1]]) - - @staticmethod - def check_boundary(state): - state[0] = (0 if state[0] < 0 else WIDTH - 1 - if state[0] > WIDTH - 1 else state[0]) - state[1] = (0 if state[1] < 0 else HEIGHT - 1 - if state[1] > HEIGHT - 1 else state[1]) - return state - - def get_transition_prob(self, state, action): - return self.transition_probability - - def get_all_states(self): - return self.all_state diff --git a/1-grid-world/2-value-iteration/value_iteration.py b/1-grid-world/2-value_iteration.py similarity index 60% rename from 1-grid-world/2-value-iteration/value_iteration.py rename to 1-grid-world/2-value_iteration.py index 8dff7281..4b31d4d7 100644 --- a/1-grid-world/2-value-iteration/value_iteration.py +++ b/1-grid-world/2-value_iteration.py @@ -1,5 +1,5 @@ -# -*- coding: utf-8 -*- -from environment import GraphicDisplay, Env +from env import GraphicDisplay, PolicyEnv as Env + class ValueIteration: def __init__(self, env): @@ -59,5 +59,40 @@ def get_value(self, state): if __name__ == "__main__": env = Env() value_iteration = ValueIteration(env) - grid_world = GraphicDisplay(value_iteration) - grid_world.mainloop() + display = GraphicDisplay(value_iteration, title="Value Iteration") + + def on_calculate(): + value_iteration.value_iteration() + display.show_values(value_iteration.value_table) + + def on_print_policy(): + # Build a policy arrow table from the greedy actions implied by V. + policy = [[[0.0] * 4 for _ in range(env.width)] for _ in range(env.height)] + for state in env.get_all_states(): + x, y = state + actions = value_iteration.get_action(state) + if not actions: + continue + prob = 1.0 / len(actions) + for a in actions: + policy[x][y][a] = prob + display.show_arrows(policy) + + def on_move(): + display.move_along_policy(value_iteration.get_action) + + def on_clear(): + value_iteration.__init__(env) + display.clear() + display.agent_pos = [0, 0] + display.clicks.clear() + + # Workflow: Calculate x several (until V stops changing) -> Print Policy -> Move. + # Print Policy unlocks after Calculate; Move unlocks after Print Policy. + display.buttons = [ + ("Calculate", on_calculate), + ("Print Policy", on_print_policy, lambda: display.click_count("Calculate") > 0), + ("Move", on_move, lambda: display.click_count("Print Policy") > 0), + ("Clear", on_clear), + ] + display.mainloop() diff --git a/1-grid-world/3-monte-carlo/environment.py b/1-grid-world/3-monte-carlo/environment.py deleted file mode 100644 index d885107d..00000000 --- a/1-grid-world/3-monte-carlo/environment.py +++ /dev/null @@ -1,113 +0,0 @@ -import time -import numpy as np -import tkinter as tk -from PIL import ImageTk, Image - -np.random.seed(1) -PhotoImage = ImageTk.PhotoImage -UNIT = 100 # pixels -HEIGHT = 5 # grid height -WIDTH = 5 # grid width - - -class Env(tk.Tk): - def __init__(self): - super(Env, self).__init__() - self.action_space = ['u', 'd', 'l', 'r'] - self.n_actions = len(self.action_space) - self.title('monte carlo') - self.geometry('{0}x{1}'.format(HEIGHT * UNIT, HEIGHT * UNIT)) - self.shapes = self.load_images() - self.canvas = self._build_canvas() - self.texts = [] - - def _build_canvas(self): - canvas = tk.Canvas(self, bg='white', - height=HEIGHT * UNIT, - width=WIDTH * UNIT) - # create grids - for c in range(0, WIDTH * UNIT, UNIT): # 0~400 by 80 - x0, y0, x1, y1 = c, 0, c, HEIGHT * UNIT - canvas.create_line(x0, y0, x1, y1) - for r in range(0, HEIGHT * UNIT, UNIT): # 0~400 by 80 - x0, y0, x1, y1 = 0, r, HEIGHT * UNIT, r - canvas.create_line(x0, y0, x1, y1) - - # add img to canvas - self.rectangle = canvas.create_image(50, 50, image=self.shapes[0]) - self.triangle1 = canvas.create_image(250, 150, image=self.shapes[1]) - self.triangle2 = canvas.create_image(150, 250, image=self.shapes[1]) - self.circle = canvas.create_image(250, 250, image=self.shapes[2]) - - # pack all - canvas.pack() - - return canvas - - def load_images(self): - rectangle = PhotoImage( - Image.open("../img/rectangle.png").resize((65, 65))) - triangle = PhotoImage( - Image.open("../img/triangle.png").resize((65, 65))) - circle = PhotoImage( - Image.open("../img/circle.png").resize((65, 65))) - - return rectangle, triangle, circle - - @staticmethod - def coords_to_state(coords): - x = int((coords[0] - 50) / 100) - y = int((coords[1] - 50) / 100) - return [x, y] - - def reset(self): - self.update() - time.sleep(0.5) - x, y = self.canvas.coords(self.rectangle) - self.canvas.move(self.rectangle, UNIT / 2 - x, UNIT / 2 - y) - # return observation - return self.coords_to_state(self.canvas.coords(self.rectangle)) - - def step(self, action): - state = self.canvas.coords(self.rectangle) - base_action = np.array([0, 0]) - self.render() - - if action == 0: # up - if state[1] > UNIT: - base_action[1] -= UNIT - elif action == 1: # down - if state[1] < (HEIGHT - 1) * UNIT: - base_action[1] += UNIT - elif action == 2: # left - if state[0] > UNIT: - base_action[0] -= UNIT - elif action == 3: # right - if state[0] < (WIDTH - 1) * UNIT: - base_action[0] += UNIT - # move agent - self.canvas.move(self.rectangle, base_action[0], base_action[1]) - # move rectangle to top level of canvas - self.canvas.tag_raise(self.rectangle) - - next_state = self.canvas.coords(self.rectangle) - - # reward function - if next_state == self.canvas.coords(self.circle): - reward = 100 - done = True - elif next_state in [self.canvas.coords(self.triangle1), - self.canvas.coords(self.triangle2)]: - reward = -100 - done = True - else: - reward = 0 - done = False - - next_state = self.coords_to_state(next_state) - - return next_state, reward, done - - def render(self): - time.sleep(0.03) - self.update() diff --git a/1-grid-world/3-monte-carlo/mc_agent.py b/1-grid-world/3-monte-carlo/mc_agent.py deleted file mode 100644 index 682b59b9..00000000 --- a/1-grid-world/3-monte-carlo/mc_agent.py +++ /dev/null @@ -1,111 +0,0 @@ -import numpy as np -import random -from collections import defaultdict -from environment import Env - - -# Monte Carlo Agent which learns every episodes from the sample -class MCAgent: - def __init__(self, actions): - self.width = 5 - self.height = 5 - self.actions = actions - self.learning_rate = 0.01 - self.discount_factor = 0.9 - self.epsilon = 0.1 - self.samples = [] - self.value_table = defaultdict(float) - - # append sample to memory(state, reward, done) - def save_sample(self, state, reward, done): - self.samples.append([state, reward, done]) - - # for every episode, agent updates q function of visited states - def update(self): - G_t = 0 - visit_state = [] - for reward in reversed(self.samples): - state = str(reward[0]) - if state not in visit_state: - visit_state.append(state) - G_t = self.discount_factor * (reward[1] + G_t) - value = self.value_table[state] - self.value_table[state] = (value + - self.learning_rate * (G_t - value)) - - # get action for the state according to the q function table - # agent pick action of epsilon-greedy policy - def get_action(self, state): - if np.random.rand() < self.epsilon: - # take random action - action = np.random.choice(self.actions) - else: - # take action according to the q function table - next_state = self.possible_next_state(state) - action = self.arg_max(next_state) - return int(action) - - # compute arg_max if multiple candidates exit, pick one randomly - @staticmethod - def arg_max(next_state): - max_index_list = [] - max_value = next_state[0] - for index, value in enumerate(next_state): - if value > max_value: - max_index_list.clear() - max_value = value - max_index_list.append(index) - elif value == max_value: - max_index_list.append(index) - return random.choice(max_index_list) - - # get the possible next states - def possible_next_state(self, state): - col, row = state - next_state = [0.0] * 4 - - if row != 0: - next_state[0] = self.value_table[str([col, row - 1])] - else: - next_state[0] = self.value_table[str(state)] - if row != self.height - 1: - next_state[1] = self.value_table[str([col, row + 1])] - else: - next_state[1] = self.value_table[str(state)] - if col != 0: - next_state[2] = self.value_table[str([col - 1, row])] - else: - next_state[2] = self.value_table[str(state)] - if col != self.width - 1: - next_state[3] = self.value_table[str([col + 1, row])] - else: - next_state[3] = self.value_table[str(state)] - - return next_state - - -# main loop -if __name__ == "__main__": - env = Env() - agent = MCAgent(actions=list(range(env.n_actions))) - - for episode in range(1000): - state = env.reset() - action = agent.get_action(state) - - while True: - env.render() - - # forward to next state. reward is number and done is boolean - next_state, reward, done = env.step(action) - agent.save_sample(next_state, reward, done) - - # get next action - action = agent.get_action(next_state) - - # at the end of each episode, update the q function table - if done: - print("episode : ", episode) - agent.update() - agent.samples.clear() - break diff --git a/1-grid-world/4-sarsa/sarsa_agent.py b/1-grid-world/3-sarsa.py similarity index 98% rename from 1-grid-world/4-sarsa/sarsa_agent.py rename to 1-grid-world/3-sarsa.py index 8a8cf9ef..5371f934 100644 --- a/1-grid-world/4-sarsa/sarsa_agent.py +++ b/1-grid-world/3-sarsa.py @@ -1,7 +1,8 @@ import numpy as np import random from collections import defaultdict -from environment import Env + +from env import Env # SARSA agent learns every time step from the sample diff --git a/1-grid-world/5-q-learning/q_learning_agent.py b/1-grid-world/4-q_learning.py similarity index 98% rename from 1-grid-world/5-q-learning/q_learning_agent.py rename to 1-grid-world/4-q_learning.py index 029c2f36..cbcb40c1 100644 --- a/1-grid-world/5-q-learning/q_learning_agent.py +++ b/1-grid-world/4-q_learning.py @@ -1,8 +1,9 @@ import numpy as np import random -from environment import Env from collections import defaultdict +from env import Env + class QLearningAgent: def __init__(self, actions): # actions = [0, 1, 2, 3] diff --git a/1-grid-world/4-sarsa/.python-version b/1-grid-world/4-sarsa/.python-version deleted file mode 100644 index 1545d966..00000000 --- a/1-grid-world/4-sarsa/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.5.0 diff --git a/1-grid-world/4-sarsa/environment.py b/1-grid-world/4-sarsa/environment.py deleted file mode 100644 index acf6d819..00000000 --- a/1-grid-world/4-sarsa/environment.py +++ /dev/null @@ -1,142 +0,0 @@ -import time -import numpy as np -import tkinter as tk -from PIL import ImageTk, Image - -np.random.seed(1) -PhotoImage = ImageTk.PhotoImage -UNIT = 100 # pixels -HEIGHT = 5 # grid height -WIDTH = 5 # grid width - - -class Env(tk.Tk): - def __init__(self): - super(Env, self).__init__() - self.action_space = ['u', 'd', 'l', 'r'] - self.n_actions = len(self.action_space) - self.title('SARSA') - self.geometry('{0}x{1}'.format(HEIGHT * UNIT, HEIGHT * UNIT)) - self.shapes = self.load_images() - self.canvas = self._build_canvas() - self.texts = [] - - def _build_canvas(self): - canvas = tk.Canvas(self, bg='white', - height=HEIGHT * UNIT, - width=WIDTH * UNIT) - # create grids - for c in range(0, WIDTH * UNIT, UNIT): # 0~400 by 80 - x0, y0, x1, y1 = c, 0, c, HEIGHT * UNIT - canvas.create_line(x0, y0, x1, y1) - for r in range(0, HEIGHT * UNIT, UNIT): # 0~400 by 80 - x0, y0, x1, y1 = 0, r, HEIGHT * UNIT, r - canvas.create_line(x0, y0, x1, y1) - - # add img to canvas - self.rectangle = canvas.create_image(50, 50, image=self.shapes[0]) - self.triangle1 = canvas.create_image(250, 150, image=self.shapes[1]) - self.triangle2 = canvas.create_image(150, 250, image=self.shapes[1]) - self.circle = canvas.create_image(250, 250, image=self.shapes[2]) - - # pack all - canvas.pack() - - return canvas - - def load_images(self): - rectangle = PhotoImage( - Image.open("../img/rectangle.png").resize((65, 65))) - triangle = PhotoImage( - Image.open("../img/triangle.png").resize((65, 65))) - circle = PhotoImage( - Image.open("../img/circle.png").resize((65, 65))) - - return rectangle, triangle, circle - - def text_value(self, row, col, contents, action, font='Helvetica', size=10, - style='normal', anchor="nw"): - if action == 0: - origin_x, origin_y = 7, 42 - elif action == 1: - origin_x, origin_y = 85, 42 - elif action == 2: - origin_x, origin_y = 42, 5 - else: - origin_x, origin_y = 42, 77 - - x, y = origin_y + (UNIT * col), origin_x + (UNIT * row) - font = (font, str(size), style) - text = self.canvas.create_text(x, y, fill="black", text=contents, - font=font, anchor=anchor) - return self.texts.append(text) - - def print_value_all(self, q_table): - for i in self.texts: - self.canvas.delete(i) - self.texts.clear() - for x in range(HEIGHT): - for y in range(WIDTH): - for action in range(0, 4): - state = [x, y] - if str(state) in q_table.keys(): - temp = q_table[str(state)][action] - self.text_value(y, x, round(temp, 2), action) - - def coords_to_state(self, coords): - x = int((coords[0] - 50) / 100) - y = int((coords[1] - 50) / 100) - return [x, y] - - def reset(self): - self.update() - time.sleep(0.5) - x, y = self.canvas.coords(self.rectangle) - self.canvas.move(self.rectangle, UNIT / 2 - x, UNIT / 2 - y) - self.render() - # return observation - return self.coords_to_state(self.canvas.coords(self.rectangle)) - - def step(self, action): - state = self.canvas.coords(self.rectangle) - base_action = np.array([0, 0]) - self.render() - - if action == 0: # up - if state[1] > UNIT: - base_action[1] -= UNIT - elif action == 1: # down - if state[1] < (HEIGHT - 1) * UNIT: - base_action[1] += UNIT - elif action == 2: # left - if state[0] > UNIT: - base_action[0] -= UNIT - elif action == 3: # right - if state[0] < (WIDTH - 1) * UNIT: - base_action[0] += UNIT - - # move agent - self.canvas.move(self.rectangle, base_action[0], base_action[1]) - # move rectangle to top level of canvas - self.canvas.tag_raise(self.rectangle) - next_state = self.canvas.coords(self.rectangle) - - # reward function - if next_state == self.canvas.coords(self.circle): - reward = 100 - done = True - elif next_state in [self.canvas.coords(self.triangle1), - self.canvas.coords(self.triangle2)]: - reward = -100 - done = True - else: - reward = 0 - done = False - - next_state = self.coords_to_state(next_state) - - return next_state, reward, done - - def render(self): - time.sleep(0.03) - self.update() diff --git a/1-grid-world/5-deep_sarsa.py b/1-grid-world/5-deep_sarsa.py new file mode 100755 index 00000000..4ab226f0 --- /dev/null +++ b/1-grid-world/5-deep_sarsa.py @@ -0,0 +1,119 @@ +"""Deep SARSA agent for the GridWorld. + +SARSA (Rummery & Niranjan, 1994) is an on-policy TD control method. +Update rule for the action-value function: + + Q(s, a) <- Q(s, a) + alpha * [r + gamma * Q(s', a') - Q(s, a)] + +The "next action a'" is the action actually taken in the next state under +the current (epsilon-greedy) policy, which makes SARSA on-policy. + +Deep SARSA replaces the table Q(s, a) with a neural network Q_theta(s), +and minimizes the squared TD error via gradient descent: + + L(theta) = ( Q_theta(s)[a] - (r + gamma * Q_theta(s')[a']) )^2 +""" +import random + +import numpy as np +import torch +import torch.nn as nn +import torch.optim as optim + +from env import DynamicEnv + +EPISODES = 1000 + + +# Approximator for Q(s, .): state in R^15 -> Q-values in R^5 (one per action). +class QNetwork(nn.Module): + def __init__(self, state_size, action_size): + super().__init__() + self.net = nn.Sequential( + nn.Linear(state_size, 30), + nn.ReLU(), + nn.Linear(30, 30), + nn.ReLU(), + nn.Linear(30, action_size), + ) + + def forward(self, x): + return self.net(x) + + +class DeepSARSAgent: + def __init__(self): + self.action_space = [0, 1, 2, 3, 4] + self.action_size = len(self.action_space) + self.state_size = 15 + self.discount_factor = 0.99 + self.learning_rate = 1e-3 + + # Epsilon-greedy exploration schedule. + self.epsilon = 1.0 + self.epsilon_decay = 0.9999 + self.epsilon_min = 0.01 + + self.model = QNetwork(self.state_size, self.action_size) + self.optimizer = optim.Adam(self.model.parameters(), lr=self.learning_rate) + self.loss_fn = nn.MSELoss() + + # Epsilon-greedy: with prob epsilon pick random, else argmax_a Q(s, a). + def get_action(self, state): + if np.random.rand() <= self.epsilon: + return random.randrange(self.action_size) + with torch.no_grad(): + q_values = self.model(torch.as_tensor(state, dtype=torch.float32)) + return int(torch.argmax(q_values).item()) + + # One-step SARSA update using the actually-taken next action a'. + def train_model(self, state, action, reward, next_state, next_action, done): + if self.epsilon > self.epsilon_min: + self.epsilon *= self.epsilon_decay + + state_t = torch.as_tensor(state, dtype=torch.float32) + next_state_t = torch.as_tensor(next_state, dtype=torch.float32) + + # Q(s, a) — keep gradient flow. + q_pred = self.model(state_t)[action] + + # TD target: r if terminal else r + gamma * Q(s', a'). + # The target is treated as a constant (no_grad), otherwise the network + # would learn to drive its own target down and training would diverge. + with torch.no_grad(): + if done: + target = torch.tensor(float(reward)) + else: + next_q = self.model(next_state_t)[next_action] + target = reward + self.discount_factor * next_q + + loss = self.loss_fn(q_pred, target) + self.optimizer.zero_grad() + loss.backward() + self.optimizer.step() + + +if __name__ == "__main__": + env = DynamicEnv(title="DeepSARSA") + agent = DeepSARSAgent() + global_step = 0 + + for e in range(EPISODES): + done = False + score = 0 + state = np.array(env.reset(), dtype=np.float32) + + while not done: + global_step += 1 + # Take an action under the current policy. + action = agent.get_action(state) + next_state, reward, done = env.step(action) + next_state = np.array(next_state, dtype=np.float32) + # SARSA needs the next action a' to form the target — sample it now. + next_action = agent.get_action(next_state) + agent.train_model(state, action, reward, next_state, next_action, done) + state = next_state + score += reward + + if done: + print(f"episode: {e} score: {score} steps: {global_step} epsilon: {agent.epsilon:.4f}") diff --git a/1-grid-world/5-q-learning/.python-version b/1-grid-world/5-q-learning/.python-version deleted file mode 100644 index 1545d966..00000000 --- a/1-grid-world/5-q-learning/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.5.0 diff --git a/1-grid-world/5-q-learning/environment.py b/1-grid-world/5-q-learning/environment.py deleted file mode 100644 index e724e5ac..00000000 --- a/1-grid-world/5-q-learning/environment.py +++ /dev/null @@ -1,148 +0,0 @@ -import time -import numpy as np -import tkinter as tk -from PIL import ImageTk, Image - -np.random.seed(1) -PhotoImage = ImageTk.PhotoImage -UNIT = 100 # pixels -HEIGHT = 5 # grid height -WIDTH = 5 # grid width - - -class Env(tk.Tk): - def __init__(self): - super(Env, self).__init__() - self.action_space = ['u', 'd', 'l', 'r'] - self.n_actions = len(self.action_space) - self.title('Q Learning') - self.geometry('{0}x{1}'.format(HEIGHT * UNIT, HEIGHT * UNIT)) - self.shapes = self.load_images() - self.canvas = self._build_canvas() - self.texts = [] - - def _build_canvas(self): - canvas = tk.Canvas(self, bg='white', - height=HEIGHT * UNIT, - width=WIDTH * UNIT) - # create grids - for c in range(0, WIDTH * UNIT, UNIT): # 0~400 by 80 - x0, y0, x1, y1 = c, 0, c, HEIGHT * UNIT - canvas.create_line(x0, y0, x1, y1) - for r in range(0, HEIGHT * UNIT, UNIT): # 0~400 by 80 - x0, y0, x1, y1 = 0, r, HEIGHT * UNIT, r - canvas.create_line(x0, y0, x1, y1) - - # add img to canvas - self.rectangle = canvas.create_image(50, 50, image=self.shapes[0]) - self.triangle1 = canvas.create_image(250, 150, image=self.shapes[1]) - self.triangle2 = canvas.create_image(150, 250, image=self.shapes[1]) - self.circle = canvas.create_image(250, 250, image=self.shapes[2]) - - # pack all - canvas.pack() - - return canvas - - def load_images(self): - rectangle = PhotoImage( - Image.open("../img/rectangle.png").resize((65, 65))) - triangle = PhotoImage( - Image.open("../img/triangle.png").resize((65, 65))) - circle = PhotoImage( - Image.open("../img/circle.png").resize((65, 65))) - - return rectangle, triangle, circle - - def text_value(self, row, col, contents, action, font='Helvetica', size=10, - style='normal', anchor="nw"): - - if action == 0: - origin_x, origin_y = 7, 42 - elif action == 1: - origin_x, origin_y = 85, 42 - elif action == 2: - origin_x, origin_y = 42, 5 - else: - origin_x, origin_y = 42, 77 - - x, y = origin_y + (UNIT * col), origin_x + (UNIT * row) - font = (font, str(size), style) - text = self.canvas.create_text(x, y, fill="black", text=contents, - font=font, anchor=anchor) - return self.texts.append(text) - - def print_value_all(self, q_table): - for i in self.texts: - self.canvas.delete(i) - self.texts.clear() - for i in range(HEIGHT): - for j in range(WIDTH): - for action in range(0, 4): - state = [i, j] - if str(state) in q_table.keys(): - temp = q_table[str(state)][action] - self.text_value(j, i, round(temp, 2), action) - - def coords_to_state(self, coords): - x = int((coords[0] - 50) / 100) - y = int((coords[1] - 50) / 100) - return [x, y] - - def state_to_coords(self, state): - x = int(state[0] * 100 + 50) - y = int(state[1] * 100 + 50) - return [x, y] - - def reset(self): - self.update() - time.sleep(0.5) - x, y = self.canvas.coords(self.rectangle) - self.canvas.move(self.rectangle, UNIT / 2 - x, UNIT / 2 - y) - self.render() - # return observation - return self.coords_to_state(self.canvas.coords(self.rectangle)) - - - def step(self, action): - state = self.canvas.coords(self.rectangle) - base_action = np.array([0, 0]) - self.render() - - if action == 0: # up - if state[1] > UNIT: - base_action[1] -= UNIT - elif action == 1: # down - if state[1] < (HEIGHT - 1) * UNIT: - base_action[1] += UNIT - elif action == 2: # left - if state[0] > UNIT: - base_action[0] -= UNIT - elif action == 3: # right - if state[0] < (WIDTH - 1) * UNIT: - base_action[0] += UNIT - - # move agent - self.canvas.move(self.rectangle, base_action[0], base_action[1]) - # move rectangle to top level of canvas - self.canvas.tag_raise(self.rectangle) - next_state = self.canvas.coords(self.rectangle) - - # reward function - if next_state == self.canvas.coords(self.circle): - reward = 100 - done = True - elif next_state in [self.canvas.coords(self.triangle1), - self.canvas.coords(self.triangle2)]: - reward = -100 - done = True - else: - reward = 0 - done = False - - next_state = self.coords_to_state(next_state) - return next_state, reward, done - - def render(self): - time.sleep(0.03) - self.update() diff --git a/1-grid-world/6-deep-sarsa/deep_sarsa_agent.py b/1-grid-world/6-deep-sarsa/deep_sarsa_agent.py deleted file mode 100755 index a1b1c23b..00000000 --- a/1-grid-world/6-deep-sarsa/deep_sarsa_agent.py +++ /dev/null @@ -1,117 +0,0 @@ -import copy -import pylab -import random -import numpy as np -from environment import Env -from keras.layers import Dense -from keras.optimizers import Adam -from keras.models import Sequential - -EPISODES = 1000 - - -# this is DeepSARSA Agent for the GridWorld -# Utilize Neural Network as q function approximator -class DeepSARSAgent: - def __init__(self): - self.load_model = False - # actions which agent can do - self.action_space = [0, 1, 2, 3, 4] - # get size of state and action - self.action_size = len(self.action_space) - self.state_size = 15 - self.discount_factor = 0.99 - self.learning_rate = 0.001 - - self.epsilon = 1. # exploration - self.epsilon_decay = .9999 - self.epsilon_min = 0.01 - self.model = self.build_model() - - if self.load_model: - self.epsilon = 0.05 - self.model.load_weights('./save_model/deep_sarsa_trained.h5') - - # approximate Q function using Neural Network - # state is input and Q Value of each action is output of network - def build_model(self): - model = Sequential() - model.add(Dense(30, input_dim=self.state_size, activation='relu')) - model.add(Dense(30, activation='relu')) - model.add(Dense(self.action_size, activation='linear')) - model.summary() - model.compile(loss='mse', optimizer=Adam(lr=self.learning_rate)) - return model - - # get action from model using epsilon-greedy policy - def get_action(self, state): - if np.random.rand() <= self.epsilon: - # The agent acts randomly - return random.randrange(self.action_size) - else: - # Predict the reward value based on the given state - state = np.float32(state) - q_values = self.model.predict(state) - return np.argmax(q_values[0]) - - def train_model(self, state, action, reward, next_state, next_action, done): - if self.epsilon > self.epsilon_min: - self.epsilon *= self.epsilon_decay - - state = np.float32(state) - next_state = np.float32(next_state) - target = self.model.predict(state)[0] - # like Q Learning, get maximum Q value at s' - # But from target model - if done: - target[action] = reward - else: - target[action] = (reward + self.discount_factor * - self.model.predict(next_state)[0][next_action]) - - target = np.reshape(target, [1, 5]) - # make minibatch which includes target q value and predicted q value - # and do the model fit! - self.model.fit(state, target, epochs=1, verbose=0) - - -if __name__ == "__main__": - env = Env() - agent = DeepSARSAgent() - - global_step = 0 - scores, episodes = [], [] - - for e in range(EPISODES): - done = False - score = 0 - state = env.reset() - state = np.reshape(state, [1, 15]) - - while not done: - # fresh env - global_step += 1 - - # get action for the current state and go one step in environment - action = agent.get_action(state) - next_state, reward, done = env.step(action) - next_state = np.reshape(next_state, [1, 15]) - next_action = agent.get_action(next_state) - agent.train_model(state, action, reward, next_state, next_action, - done) - state = next_state - # every time step we do training - score += reward - - state = copy.deepcopy(next_state) - - if done: - scores.append(score) - episodes.append(e) - pylab.plot(episodes, scores, 'b') - pylab.savefig("./save_graph/deep_sarsa_.png") - print("episode:", e, " score:", score, "global_step", - global_step, " epsilon:", agent.epsilon) - - if e % 100 == 0: - agent.model.save_weights("./save_model/deep_sarsa.h5") diff --git a/1-grid-world/6-deep-sarsa/environment.py b/1-grid-world/6-deep-sarsa/environment.py deleted file mode 100755 index c390de8b..00000000 --- a/1-grid-world/6-deep-sarsa/environment.py +++ /dev/null @@ -1,240 +0,0 @@ -import time -import numpy as np -import tkinter as tk -from PIL import ImageTk, Image - -PhotoImage = ImageTk.PhotoImage -UNIT = 50 # pixels -HEIGHT = 5 # grid height -WIDTH = 5 # grid width - -np.random.seed(1) - - -class Env(tk.Tk): - def __init__(self): - super(Env, self).__init__() - self.action_space = ['u', 'd', 'l', 'r'] - self.action_size = len(self.action_space) - self.title('DeepSARSA') - self.geometry('{0}x{1}'.format(HEIGHT * UNIT, HEIGHT * UNIT)) - self.shapes = self.load_images() - self.canvas = self._build_canvas() - self.counter = 0 - self.rewards = [] - self.goal = [] - # obstacle - self.set_reward([0, 1], -1) - self.set_reward([1, 2], -1) - self.set_reward([2, 3], -1) - # #goal - self.set_reward([4, 4], 1) - - def _build_canvas(self): - canvas = tk.Canvas(self, bg='white', - height=HEIGHT * UNIT, - width=WIDTH * UNIT) - # create grids - for c in range(0, WIDTH * UNIT, UNIT): # 0~400 by 80 - x0, y0, x1, y1 = c, 0, c, HEIGHT * UNIT - canvas.create_line(x0, y0, x1, y1) - for r in range(0, HEIGHT * UNIT, UNIT): # 0~400 by 80 - x0, y0, x1, y1 = 0, r, HEIGHT * UNIT, r - canvas.create_line(x0, y0, x1, y1) - - self.rewards = [] - self.goal = [] - # add image to canvas - x, y = UNIT/2, UNIT/2 - self.rectangle = canvas.create_image(x, y, image=self.shapes[0]) - - # pack all` - canvas.pack() - - return canvas - - def load_images(self): - rectangle = PhotoImage( - Image.open("../img/rectangle.png").resize((30, 30))) - triangle = PhotoImage( - Image.open("../img/triangle.png").resize((30, 30))) - circle = PhotoImage( - Image.open("../img/circle.png").resize((30, 30))) - - return rectangle, triangle, circle - - def reset_reward(self): - - for reward in self.rewards: - self.canvas.delete(reward['figure']) - - self.rewards.clear() - self.goal.clear() - self.set_reward([0, 1], -1) - self.set_reward([1, 2], -1) - self.set_reward([2, 3], -1) - - # #goal - self.set_reward([4, 4], 1) - - def set_reward(self, state, reward): - state = [int(state[0]), int(state[1])] - x = int(state[0]) - y = int(state[1]) - temp = {} - if reward > 0: - temp['reward'] = reward - temp['figure'] = self.canvas.create_image((UNIT * x) + UNIT / 2, - (UNIT * y) + UNIT / 2, - image=self.shapes[2]) - - self.goal.append(temp['figure']) - - - elif reward < 0: - temp['direction'] = -1 - temp['reward'] = reward - temp['figure'] = self.canvas.create_image((UNIT * x) + UNIT / 2, - (UNIT * y) + UNIT / 2, - image=self.shapes[1]) - - temp['coords'] = self.canvas.coords(temp['figure']) - temp['state'] = state - self.rewards.append(temp) - - # new methods - - def check_if_reward(self, state): - check_list = dict() - check_list['if_goal'] = False - rewards = 0 - - for reward in self.rewards: - if reward['state'] == state: - rewards += reward['reward'] - if reward['reward'] == 1: - check_list['if_goal'] = True - - check_list['rewards'] = rewards - - return check_list - - def coords_to_state(self, coords): - x = int((coords[0] - UNIT / 2) / UNIT) - y = int((coords[1] - UNIT / 2) / UNIT) - return [x, y] - - def reset(self): - self.update() - time.sleep(0.5) - x, y = self.canvas.coords(self.rectangle) - self.canvas.move(self.rectangle, UNIT / 2 - x, UNIT / 2 - y) - # return observation - self.reset_reward() - return self.get_state() - - def step(self, action): - self.counter += 1 - self.render() - - if self.counter % 2 == 1: - self.rewards = self.move_rewards() - - next_coords = self.move(self.rectangle, action) - check = self.check_if_reward(self.coords_to_state(next_coords)) - done = check['if_goal'] - reward = check['rewards'] - - self.canvas.tag_raise(self.rectangle) - - s_ = self.get_state() - - return s_, reward, done - - def get_state(self): - - location = self.coords_to_state(self.canvas.coords(self.rectangle)) - agent_x = location[0] - agent_y = location[1] - - states = list() - - # locations.append(agent_x) - # locations.append(agent_y) - - for reward in self.rewards: - reward_location = reward['state'] - states.append(reward_location[0] - agent_x) - states.append(reward_location[1] - agent_y) - if reward['reward'] < 0: - states.append(-1) - states.append(reward['direction']) - else: - states.append(1) - - return states - - def move_rewards(self): - new_rewards = [] - for temp in self.rewards: - if temp['reward'] == 1: - new_rewards.append(temp) - continue - temp['coords'] = self.move_const(temp) - temp['state'] = self.coords_to_state(temp['coords']) - new_rewards.append(temp) - return new_rewards - - def move_const(self, target): - - s = self.canvas.coords(target['figure']) - - base_action = np.array([0, 0]) - - if s[0] == (WIDTH - 1) * UNIT + UNIT / 2: - target['direction'] = 1 - elif s[0] == UNIT / 2: - target['direction'] = -1 - - if target['direction'] == -1: - base_action[0] += UNIT - elif target['direction'] == 1: - base_action[0] -= UNIT - - if (target['figure'] is not self.rectangle - and s == [(WIDTH - 1) * UNIT, (HEIGHT - 1) * UNIT]): - base_action = np.array([0, 0]) - - self.canvas.move(target['figure'], base_action[0], base_action[1]) - - s_ = self.canvas.coords(target['figure']) - - return s_ - - def move(self, target, action): - s = self.canvas.coords(target) - - base_action = np.array([0, 0]) - - if action == 0: # up - if s[1] > UNIT: - base_action[1] -= UNIT - elif action == 1: # down - if s[1] < (HEIGHT - 1) * UNIT: - base_action[1] += UNIT - elif action == 2: # right - if s[0] < (WIDTH - 1) * UNIT: - base_action[0] += UNIT - elif action == 3: # left - if s[0] > UNIT: - base_action[0] -= UNIT - - self.canvas.move(target, base_action[0], base_action[1]) - - s_ = self.canvas.coords(target) - - return s_ - - def render(self): - time.sleep(0.07) - self.update() diff --git a/1-grid-world/6-deep-sarsa/save_graph/deep_sarsa_trained.png b/1-grid-world/6-deep-sarsa/save_graph/deep_sarsa_trained.png deleted file mode 100644 index 8dec1d06..00000000 Binary files a/1-grid-world/6-deep-sarsa/save_graph/deep_sarsa_trained.png and /dev/null differ diff --git a/1-grid-world/6-deep-sarsa/save_model/deep_sarsa_trained.h5 b/1-grid-world/6-deep-sarsa/save_model/deep_sarsa_trained.h5 deleted file mode 100644 index 23ba39c9..00000000 Binary files a/1-grid-world/6-deep-sarsa/save_model/deep_sarsa_trained.h5 and /dev/null differ diff --git a/1-grid-world/6-reinforce.py b/1-grid-world/6-reinforce.py new file mode 100644 index 00000000..f072276c --- /dev/null +++ b/1-grid-world/6-reinforce.py @@ -0,0 +1,129 @@ +"""REINFORCE (Monte-Carlo policy gradient) agent for the GridWorld. + +Williams, 1992: "Simple Statistical Gradient-Following Algorithms for +Connectionist Reinforcement Learning". + +Policy gradient theorem: + + grad_theta J(theta) = E_pi [ grad_theta log pi_theta(a|s) * G_t ] + +where G_t = sum_{k>=t} gamma^(k-t) * r_k is the return from step t. + +We use the per-episode Monte-Carlo estimator: collect a full trajectory, +compute discounted returns G_t, then ascend the gradient. The returns are +standardized (zero-mean, unit-variance) as a simple variance-reduction +trick (acts like a constant baseline). + +Implementation note: we maximize expected return, i.e. minimize the +negative log-likelihood weighted by G_t. +""" +import numpy as np +import torch +import torch.nn as nn +import torch.optim as optim + +from env import DynamicEnv + +EPISODES = 2500 + + +# Policy network: state -> logits over actions. +# Softmax is applied where we need probabilities (sampling / log-prob). +class PolicyNetwork(nn.Module): + def __init__(self, state_size, action_size): + super().__init__() + self.net = nn.Sequential( + nn.Linear(state_size, 24), + nn.ReLU(), + nn.Linear(24, 24), + nn.ReLU(), + nn.Linear(24, action_size), + ) + + def forward(self, x): + return self.net(x) + + +class ReinforceAgent: + def __init__(self): + self.action_space = [0, 1, 2, 3, 4] + self.action_size = len(self.action_space) + self.state_size = 15 + self.discount_factor = 0.99 + self.learning_rate = 1e-3 + + self.model = PolicyNetwork(self.state_size, self.action_size) + self.optimizer = optim.Adam(self.model.parameters(), lr=self.learning_rate) + # Per-episode trajectory buffer. + self.states, self.actions, self.rewards = [], [], [] + + # Sample a ~ pi_theta(.|s). + def get_action(self, state): + with torch.no_grad(): + logits = self.model(torch.as_tensor(state, dtype=torch.float32)) + probs = torch.softmax(logits, dim=-1).numpy() + return int(np.random.choice(self.action_size, p=probs)) + + # G_t = r_t + gamma * G_{t+1}, computed backwards from the episode end. + def discount_rewards(self, rewards): + discounted = np.zeros_like(rewards, dtype=np.float32) + running = 0.0 + for t in reversed(range(len(rewards))): + running = running * self.discount_factor + rewards[t] + discounted[t] = running + return discounted + + def append_sample(self, state, action, reward): + self.states.append(state) + self.actions.append(action) + self.rewards.append(reward) + + # Single gradient step using the whole episode. + def train_model(self): + returns = self.discount_rewards(np.array(self.rewards, dtype=np.float32)) + # Variance-reduction baseline (standardization). + returns = (returns - returns.mean()) / (returns.std() + 1e-8) + + states = torch.as_tensor(np.array(self.states), dtype=torch.float32) + actions = torch.as_tensor(self.actions, dtype=torch.long) + returns_t = torch.as_tensor(returns, dtype=torch.float32) + + # log pi_theta(a_t | s_t) for each step in the trajectory. + logits = self.model(states) + log_probs = torch.log_softmax(logits, dim=-1) + chosen = log_probs.gather(1, actions.unsqueeze(1)).squeeze(1) + # Negative log-likelihood weighted by return -> minimize == ascend policy gradient. + loss = -(chosen * returns_t).sum() + + self.optimizer.zero_grad() + loss.backward() + self.optimizer.step() + + self.states, self.actions, self.rewards = [], [], [] + + +if __name__ == "__main__": + # REINFORCE uses a per-step -0.1 penalty to encourage shorter paths. + env = DynamicEnv(title="REINFORCE", step_penalty=0.1) + agent = ReinforceAgent() + global_step = 0 + + for e in range(EPISODES): + done = False + score = 0 + state = np.array(env.reset(), dtype=np.float32) + + while not done: + global_step += 1 + action = agent.get_action(state) + next_state, reward, done = env.step(action) + next_state = np.array(next_state, dtype=np.float32) + + agent.append_sample(state, action, reward) + score += reward + state = next_state + + if done: + # REINFORCE updates once per episode (Monte-Carlo). + agent.train_model() + print(f"episode: {e} score: {round(score, 2)} steps: {global_step}") diff --git a/1-grid-world/7-reinforce/environment.py b/1-grid-world/7-reinforce/environment.py deleted file mode 100644 index c8283baa..00000000 --- a/1-grid-world/7-reinforce/environment.py +++ /dev/null @@ -1,239 +0,0 @@ -import time -import numpy as np -import tkinter as tk -from PIL import ImageTk, Image - -PhotoImage = ImageTk.PhotoImage -UNIT = 50 # pixels -HEIGHT = 5 # grid height -WIDTH = 5 # grid width - -np.random.seed(1) - - -class Env(tk.Tk): - def __init__(self): - super(Env, self).__init__() - self.action_space = ['u', 'd', 'l', 'r'] - self.action_size = len(self.action_space) - self.title('Reinforce') - self.geometry('{0}x{1}'.format(HEIGHT * UNIT, HEIGHT * UNIT)) - self.shapes = self.load_images() - self.canvas = self._build_canvas() - self.counter = 0 - self.rewards = [] - self.goal = [] - # obstacle - self.set_reward([0, 1], -1) - self.set_reward([1, 2], -1) - self.set_reward([2, 3], -1) - # #goal - self.set_reward([4, 4], 1) - - def _build_canvas(self): - canvas = tk.Canvas(self, bg='white', - height=HEIGHT * UNIT, - width=WIDTH * UNIT) - # create grids - for c in range(0, WIDTH * UNIT, UNIT): # 0~400 by 80 - x0, y0, x1, y1 = c, 0, c, HEIGHT * UNIT - canvas.create_line(x0, y0, x1, y1) - for r in range(0, HEIGHT * UNIT, UNIT): # 0~400 by 80 - x0, y0, x1, y1 = 0, r, HEIGHT * UNIT, r - canvas.create_line(x0, y0, x1, y1) - - self.rewards = [] - self.goal = [] - # add image to canvas - x, y = UNIT/2, UNIT/2 - self.rectangle = canvas.create_image(x, y, image=self.shapes[0]) - - # pack all` - canvas.pack() - - return canvas - - def load_images(self): - rectangle = PhotoImage( - Image.open("../img/rectangle.png").resize((30, 30))) - triangle = PhotoImage( - Image.open("../img/triangle.png").resize((30, 30))) - circle = PhotoImage( - Image.open("../img/circle.png").resize((30, 30))) - - return rectangle, triangle, circle - - def reset_reward(self): - - for reward in self.rewards: - self.canvas.delete(reward['figure']) - - self.rewards.clear() - self.goal.clear() - self.set_reward([0, 1], -1) - self.set_reward([1, 2], -1) - self.set_reward([2, 3], -1) - - # #goal - self.set_reward([4, 4], 1) - - def set_reward(self, state, reward): - state = [int(state[0]), int(state[1])] - x = int(state[0]) - y = int(state[1]) - temp = {} - if reward > 0: - temp['reward'] = reward - temp['figure'] = self.canvas.create_image((UNIT * x) + UNIT / 2, - (UNIT * y) + UNIT / 2, - image=self.shapes[2]) - - self.goal.append(temp['figure']) - - - elif reward < 0: - temp['direction'] = -1 - temp['reward'] = reward - temp['figure'] = self.canvas.create_image((UNIT * x) + UNIT / 2, - (UNIT * y) + UNIT / 2, - image=self.shapes[1]) - - temp['coords'] = self.canvas.coords(temp['figure']) - temp['state'] = state - self.rewards.append(temp) - - # new methods - - def check_if_reward(self, state): - check_list = dict() - check_list['if_goal'] = False - rewards = 0 - - for reward in self.rewards: - if reward['state'] == state: - rewards += reward['reward'] - if reward['reward'] > 0: - check_list['if_goal'] = True - - check_list['rewards'] = rewards - - return check_list - - def coords_to_state(self, coords): - x = int((coords[0] - UNIT / 2) / UNIT) - y = int((coords[1] - UNIT / 2) / UNIT) - return [x, y] - - def reset(self): - self.update() - x, y = self.canvas.coords(self.rectangle) - self.canvas.move(self.rectangle, UNIT / 2 - x, UNIT / 2 - y) - # return observation - self.reset_reward() - return self.get_state() - - def step(self, action): - self.counter += 1 - self.render() - - if self.counter % 2 == 1: - self.rewards = self.move_rewards() - - next_coords = self.move(self.rectangle, action) - check = self.check_if_reward(self.coords_to_state(next_coords)) - done = check['if_goal'] - reward = check['rewards'] - reward -= 0.1 - self.canvas.tag_raise(self.rectangle) - - s_ = self.get_state() - - return s_, reward, done - - def get_state(self): - - location = self.coords_to_state(self.canvas.coords(self.rectangle)) - agent_x = location[0] - agent_y = location[1] - - states = list() - - # locations.append(agent_x) - # locations.append(agent_y) - - for reward in self.rewards: - reward_location = reward['state'] - states.append(reward_location[0] - agent_x) - states.append(reward_location[1] - agent_y) - if reward['reward'] < 0: - states.append(-1) - states.append(reward['direction']) - else: - states.append(1) - - return states - - def move_rewards(self): - new_rewards = [] - for temp in self.rewards: - if temp['reward'] > 0: - new_rewards.append(temp) - continue - temp['coords'] = self.move_const(temp) - temp['state'] = self.coords_to_state(temp['coords']) - new_rewards.append(temp) - return new_rewards - - def move_const(self, target): - - s = self.canvas.coords(target['figure']) - - base_action = np.array([0, 0]) - - if s[0] == (WIDTH - 1) * UNIT + UNIT / 2: - target['direction'] = 1 - elif s[0] == UNIT / 2: - target['direction'] = -1 - - if target['direction'] == -1: - base_action[0] += UNIT - elif target['direction'] == 1: - base_action[0] -= UNIT - - if (target['figure'] is not self.rectangle - and s == [(WIDTH - 1) * UNIT, (HEIGHT - 1) * UNIT]): - base_action = np.array([0, 0]) - - self.canvas.move(target['figure'], base_action[0], base_action[1]) - - s_ = self.canvas.coords(target['figure']) - - return s_ - - def move(self, target, action): - s = self.canvas.coords(target) - - base_action = np.array([0, 0]) - - if action == 0: # up - if s[1] > UNIT: - base_action[1] -= UNIT - elif action == 1: # down - if s[1] < (HEIGHT - 1) * UNIT: - base_action[1] += UNIT - elif action == 2: # right - if s[0] < (WIDTH - 1) * UNIT: - base_action[0] += UNIT - elif action == 3: # left - if s[0] > UNIT: - base_action[0] -= UNIT - - self.canvas.move(target, base_action[0], base_action[1]) - - s_ = self.canvas.coords(target) - - return s_ - - def render(self): - time.sleep(0.07) - self.update() diff --git a/1-grid-world/7-reinforce/reinforce_agent.py b/1-grid-world/7-reinforce/reinforce_agent.py deleted file mode 100644 index 2a37c851..00000000 --- a/1-grid-world/7-reinforce/reinforce_agent.py +++ /dev/null @@ -1,129 +0,0 @@ -import copy -import pylab -import numpy as np -from environment import Env -from keras.layers import Dense -from keras.optimizers import Adam -from keras.models import Sequential -from keras import backend as K - -EPISODES = 2500 - - -# this is REINFORCE Agent for GridWorld -class ReinforceAgent: - def __init__(self): - self.load_model = True - # actions which agent can do - self.action_space = [0, 1, 2, 3, 4] - # get size of state and action - self.action_size = len(self.action_space) - self.state_size = 15 - self.discount_factor = 0.99 - self.learning_rate = 0.001 - - self.model = self.build_model() - self.optimizer = self.optimizer() - self.states, self.actions, self.rewards = [], [], [] - - if self.load_model: - self.model.load_weights('./save_model/reinforce_trained.h5') - - # state is input and probability of each action(policy) is output of network - def build_model(self): - model = Sequential() - model.add(Dense(24, input_dim=self.state_size, activation='relu')) - model.add(Dense(24, activation='relu')) - model.add(Dense(self.action_size, activation='softmax')) - model.summary() - return model - - # create error function and training function to update policy network - def optimizer(self): - action = K.placeholder(shape=[None, 5]) - discounted_rewards = K.placeholder(shape=[None, ]) - - # Calculate cross entropy error function - action_prob = K.sum(action * self.model.output, axis=1) - cross_entropy = K.log(action_prob) * discounted_rewards - loss = -K.sum(cross_entropy) - - # create training function - optimizer = Adam(lr=self.learning_rate) - updates = optimizer.get_updates(self.model.trainable_weights, [], - loss) - train = K.function([self.model.input, action, discounted_rewards], [], - updates=updates) - - return train - - # get action from policy network - def get_action(self, state): - policy = self.model.predict(state)[0] - return np.random.choice(self.action_size, 1, p=policy)[0] - - # calculate discounted rewards - def discount_rewards(self, rewards): - discounted_rewards = np.zeros_like(rewards) - running_add = 0 - for t in reversed(range(0, len(rewards))): - running_add = running_add * self.discount_factor + rewards[t] - discounted_rewards[t] = running_add - return discounted_rewards - - # save states, actions and rewards for an episode - def append_sample(self, state, action, reward): - self.states.append(state[0]) - self.rewards.append(reward) - act = np.zeros(self.action_size) - act[action] = 1 - self.actions.append(act) - - # update policy neural network - def train_model(self): - discounted_rewards = np.float32(self.discount_rewards(self.rewards)) - discounted_rewards -= np.mean(discounted_rewards) - discounted_rewards /= np.std(discounted_rewards) - - self.optimizer([self.states, self.actions, discounted_rewards]) - self.states, self.actions, self.rewards = [], [], [] - - -if __name__ == "__main__": - env = Env() - agent = ReinforceAgent() - - global_step = 0 - scores, episodes = [], [] - - for e in range(EPISODES): - done = False - score = 0 - # fresh env - state = env.reset() - state = np.reshape(state, [1, 15]) - - while not done: - global_step += 1 - # get action for the current state and go one step in environment - action = agent.get_action(state) - next_state, reward, done = env.step(action) - next_state = np.reshape(next_state, [1, 15]) - - agent.append_sample(state, action, reward) - score += reward - state = copy.deepcopy(next_state) - - if done: - # update policy neural network for each episode - agent.train_model() - scores.append(score) - episodes.append(e) - score = round(score, 2) - print("episode:", e, " score:", score, " time_step:", - global_step) - - if e % 100 == 0: - pylab.plot(episodes, scores, 'b') - pylab.savefig("./save_graph/reinforce.png") - agent.model.save_weights("./save_model/reinforce.h5") diff --git a/1-grid-world/7-reinforce/save_graph/reinforce_trained.png b/1-grid-world/7-reinforce/save_graph/reinforce_trained.png deleted file mode 100644 index 3be9edb7..00000000 Binary files a/1-grid-world/7-reinforce/save_graph/reinforce_trained.png and /dev/null differ diff --git a/1-grid-world/7-reinforce/save_model/reinforce_trained.h5 b/1-grid-world/7-reinforce/save_model/reinforce_trained.h5 deleted file mode 100644 index cb206f51..00000000 Binary files a/1-grid-world/7-reinforce/save_model/reinforce_trained.h5 and /dev/null differ diff --git a/1-grid-world/README.md b/1-grid-world/README.md deleted file mode 100644 index a955308e..00000000 --- a/1-grid-world/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# Grid World with Reinforcement Learning -This is Grid World example that we made for the simple algorithm test -The game is simple. The red rectangle must arrive in the circle, avoiding triangle. - -

- -
- - - -## Dynamic Programming -**1. Policy Iteration** - -**2. Value Iteration** - -
- -## Reinforcement Learning Fundamental Algorithms -**3. Monte-Carlo** - -**4. SARSA** - -**5. Q-Learning** - -
- -## Futher Reinforcement Learning Algorithms ->we have changed Grid World so the obstacles are moving. To solve this problem, we have to use function approximator. -We used Neural Network as function approximator - -

- -
- -**6. DQN** - -**7. Policy Gradient** - - diff --git a/1-grid-world/env.py b/1-grid-world/env.py new file mode 100644 index 00000000..840a3e34 --- /dev/null +++ b/1-grid-world/env.py @@ -0,0 +1,415 @@ +"""Pygame grid-world envs and DP viewer (shared by all six algorithms).""" +import math +import time + +import pygame + +UNIT, DYN_UNIT, WIDTH, HEIGHT, FPS_DELAY = 100, 100, 5, 5, 0.03 +WHITE, BLACK, GRID_LINE = (255, 255, 255), (0, 0, 0), (200, 200, 200) +AGENT_COLOR, OBSTACLE_COLOR, GOAL_COLOR = (60, 120, 220), (220, 60, 60), (60, 200, 100) +TEXT_COLOR = (40, 40, 40) + +# 0=up, 1=down, 2=left, 3=right for DP / static Env (state = [row, col] for DP, +# [col, row] for Env — see each class). DynamicEnv uses 0=up,1=down,2=right,3=left. +DP_ACTIONS = [(-1, 0), (1, 0), (0, -1), (0, 1)] + + +def _pump_events(): + for event in pygame.event.get(): + if event.type == pygame.QUIT: + pygame.quit() + raise SystemExit + + +def _open(title, size): + pygame.init() + pygame.display.set_caption(title) + return pygame.display.set_mode(size) + + +def _grid_lines(surf, unit, y_off=0): + for c in range(WIDTH + 1): + pygame.draw.line(surf, GRID_LINE, (c * unit, y_off), (c * unit, y_off + HEIGHT * unit)) + for r in range(HEIGHT + 1): + pygame.draw.line(surf, GRID_LINE, (0, y_off + r * unit), (WIDTH * unit, y_off + r * unit)) + + +def _center(x, y, unit, y_off=0): + return x * unit + unit // 2, y_off + y * unit + unit // 2 + + +def _square(surf, x, y, unit, color, y_off=0, fill=True): + cx, cy = _center(x, y, unit, y_off) + s = int(unit * 0.65) + rect = pygame.Rect(cx - s // 2, cy - s // 2, s, s) + pygame.draw.rect(surf, color, rect, 0 if fill else 3) + return rect + + +def _circle(surf, x, y, unit, color, y_off=0): + pygame.draw.circle(surf, color, _center(x, y, unit, y_off), int(unit * 0.33)) + + +def _triangle(surf, x, y, unit, color, y_off=0): + cx, cy = _center(x, y, unit, y_off) + r = int(unit * 0.36) + pygame.draw.polygon(surf, color, [(cx, cy - r), (cx - r, cy + r), (cx + r, cy + r)]) + + +# --------------------------------------------------------------------------- +class Env: + """Static 5x5 grid for tabular SARSA / Q-learning. agent=[col,row].""" + n_actions = 4 + HUD = 32 + + def __init__(self, title="GridWorld"): + self.title = title + self.agent = [0, 0] + self.obstacles = [[1, 2], [2, 1]] + self.goal = [2, 2] + self.q_overlay = None # set by print_value_all; rendered on next render() + self.episode = 0 # current episode index + self.steps = 0 # steps taken in the current episode + self.last_reward = None # terminal reward from the previous episode + self._screen = None + + def reset(self): + # Open a new episode (skipped at the very first reset). + if self.steps > 0: + self.episode += 1 + self.agent = [0, 0] + self.steps = 0 + if self._screen is not None: + self.render() + time.sleep(0.3) + return list(self.agent) + + def step(self, action): + x, y = self.agent + if action == 0 and y > 0: y -= 1 + elif action == 1 and y < HEIGHT - 1: y += 1 + elif action == 2 and x > 0: x -= 1 + elif action == 3 and x < WIDTH - 1: x += 1 + self.agent = [x, y] + self.steps += 1 + if self.agent == self.goal: + self.last_reward = 100 + return list(self.agent), 100, True + if self.agent in self.obstacles: + self.last_reward = -100 + return list(self.agent), -100, True + return list(self.agent), 0, False + + def print_value_all(self, q_table): + self.q_overlay = q_table + + def render(self): + if self._screen is None: + self._screen = _open(self.title, (WIDTH * UNIT, HEIGHT * UNIT + self.HUD)) + self._font = pygame.font.SysFont(None, 18) + self._hud_font = pygame.font.SysFont(None, 22) + _pump_events() + s = self._screen + hud = self.HUD + s.fill(WHITE) + # HUD bar. + pygame.draw.rect(s, (30, 30, 30), pygame.Rect(0, 0, WIDTH * UNIT, hud)) + # Steps display rounds down to the nearest 5 so the number doesn't + # flicker every frame (real step count is still in self.steps). + steps_shown = (self.steps // 5) * 5 + last = f"{self.last_reward:+d}" if self.last_reward is not None else "—" + t = self._hud_font.render( + f"Episode: {self.episode} Steps: {steps_shown} Last Score: {last}", + True, (240, 240, 240)) + s.blit(t, (8, (hud - t.get_height()) // 2)) + # Grid + landmarks. + _grid_lines(s, UNIT, y_off=hud) + for ox, oy in self.obstacles: + _triangle(s, ox, oy, UNIT, OBSTACLE_COLOR, y_off=hud) + _circle(s, *self.goal, unit=UNIT, color=GOAL_COLOR, y_off=hud) + _square(s, *self.agent, unit=UNIT, color=AGENT_COLOR, y_off=hud) + if self.q_overlay is not None: + offsets = [(0, -UNIT // 2 + 10), (0, UNIT // 2 - 10), + (-UNIT // 2 + 15, 0), (UNIT // 2 - 15, 0)] + for x in range(WIDTH): + for y in range(HEIGHT): + qs = self.q_overlay.get(str([x, y])) + if qs is None: continue + cx, cy = _center(x, y, UNIT, y_off=hud) + for i, q in enumerate(qs): + t = self._font.render(f"{q:+.2f}", True, TEXT_COLOR) + s.blit(t, t.get_rect(center=(cx + offsets[i][0], cy + offsets[i][1]))) + pygame.display.flip() + time.sleep(FPS_DELAY) + + +# --------------------------------------------------------------------------- +class DynamicEnv: + """5x5 grid with horizontally-bouncing obstacles (Deep SARSA, REINFORCE). + + Goal at (4,4) terminates; obstacle hit costs -1 but continues. + State: 15-dim relative encoding (4 per obstacle, 3 for goal). + Actions: 0=up, 1=down, 2=right, 3=left (note: differs from Env). + """ + n_actions = 4 + state_size = 15 + HUD = 32 + + def __init__(self, title="DynamicGridWorld", step_penalty=0.0, render_mode="human"): + self.title, self.step_penalty, self.render_mode = title, step_penalty, render_mode + self.agent = [0, 0] + self.obstacles_init = [[0, 1], [1, 2], [2, 3]] + self.goal = [4, 4] + self.obstacles = [] + self.counter, self.episode, self.score, self._hit = 0, 0, 0.0, 0 + self.last_score = None + self._screen = None + + def reset(self): + # Capture the prior episode's final score before wiping. + if self.counter > 0: + self.last_score = self.score + self.episode += 1 + self.agent, self.counter, self.score, self._hit = [0, 0], 0, 0.0, 0 + self.obstacles = [{"state": list(p), "direction": -1} for p in self.obstacles_init] + if self._screen is not None: + self.render(); time.sleep(0.3) + return self._state() + + def step(self, action): + self.counter += 1 + self.render() + if self.counter % 2 == 1: + for o in self.obstacles: + if o["state"][0] == WIDTH - 1: o["direction"] = 1 + elif o["state"][0] == 0: o["direction"] = -1 + o["state"][0] += 1 if o["direction"] == -1 else -1 + + x, y = self.agent + if action == 0 and y > 0: y -= 1 + elif action == 1 and y < HEIGHT - 1: y += 1 + elif action == 2 and x < WIDTH - 1: x += 1 + elif action == 3 and x > 0: x -= 1 + self.agent = [x, y] + + done = self.agent == self.goal + reward = 1.0 if done else sum(-1.0 for o in self.obstacles if o["state"] == self.agent) + reward -= self.step_penalty + self.score += reward + if reward < -self.step_penalty: + self._hit = 4 + return self._state(), reward, done + + def _state(self): + ax, ay = self.agent + s = [] + for o in self.obstacles: + ox, oy = o["state"] + s += [ox - ax, oy - ay, -1, o["direction"]] + s += [self.goal[0] - ax, self.goal[1] - ay, 1] + return s + + def render(self): + if self.render_mode is None: return + if self._screen is None: + self._screen = _open(self.title, (WIDTH * DYN_UNIT, HEIGHT * DYN_UNIT + self.HUD)) + self._hud_font = pygame.font.SysFont(None, 22) + self._popup_font = pygame.font.SysFont(None, 28) + _pump_events() + s, hud = self._screen, self.HUD + s.fill(WHITE) + pygame.draw.rect(s, (30, 30, 30), pygame.Rect(0, 0, WIDTH * DYN_UNIT, hud)) + last = f"{self.last_score:+.1f}" if self.last_score is not None else "—" + t = self._hud_font.render( + f"Episode: {self.episode} Score: {self.score:+.1f} Last Score: {last}", + True, (240, 240, 240)) + s.blit(t, (8, (hud - t.get_height()) // 2)) + _grid_lines(s, DYN_UNIT, y_off=hud) + _circle(s, *self.goal, unit=DYN_UNIT, color=GOAL_COLOR, y_off=hud) + for o in self.obstacles: + _triangle(s, *o["state"], unit=DYN_UNIT, color=OBSTACLE_COLOR, y_off=hud) + rect = _square(s, *self.agent, unit=DYN_UNIT, + color=OBSTACLE_COLOR if self._hit > 0 else AGENT_COLOR, y_off=hud) + if self._hit > 0: + p = self._popup_font.render("-1", True, OBSTACLE_COLOR) + s.blit(p, p.get_rect(center=(rect.centerx, rect.top - 14))) + self._hit -= 1 + pygame.display.flip() + time.sleep(FPS_DELAY) + + +# --------------------------------------------------------------------------- +class PolicyEnv: + """Pure-data MDP for policy/value iteration. state = [row, col].""" + transition_probability = 1 + possible_actions = [0, 1, 2, 3] + + def __init__(self): + self.width, self.height = WIDTH, HEIGHT + self.reward = [[0.0] * WIDTH for _ in range(HEIGHT)] + self.reward[2][2], self.reward[1][2], self.reward[2][1] = 1.0, -1.0, -1.0 + self.all_state = [[x, y] for x in range(WIDTH) for y in range(HEIGHT)] + + def get_all_states(self): + return self.all_state + + def state_after_action(self, state, action): + dx, dy = DP_ACTIONS[action] + return [max(0, min(WIDTH - 1, state[0] + dx)), max(0, min(HEIGHT - 1, state[1] + dy))] + + def get_reward(self, state, action): + ns = self.state_after_action(state, action) + return self.reward[ns[0]][ns[1]] + + def get_transition_prob(self, state, action): + return self.transition_probability + + +# --------------------------------------------------------------------------- +class GraphicDisplay: + """Pygame button-driven viewer for policy / value iteration. + + Set `display.buttons = [(label, handler[, enabled]), ...]` (up to 4). + `enabled` is an optional zero-arg callable returning bool; when it + returns False the button is greyed out and clicks are ignored. + `show_values(V)` / `show_arrows(policy_table)` overlay; `clear()` + removes them. `move_along_policy(picker)` animates greedy moves. + """ + BAR = 50 + + def __init__(self, agent, title, buttons=None): + self.agent = agent + self.env = PolicyEnv() + self.title = title + self.buttons = buttons or [] + self.agent_pos = [0, 0] + # Per-label click counts, available to button `enabled` predicates. + self.clicks = {} + # Brief "pressed" flash so clicks feel responsive. + self._press_label = None + self._press_frames = 0 + self._screen = None + self._values = None + self._arrows = None + + def click_count(self, label): + return self.clicks.get(label, 0) + + def show_values(self, v): self._values = v + def show_arrows(self, p): self._arrows = p + def clear(self): self._values = self._arrows = None + + def move_along_policy(self, picker): + self.agent_pos = [0, 0] + while True: + self._render(); pygame.time.wait(200) + r, c = self.agent_pos + # Stop at the goal cell — picker may not be defined there + # (policy iteration's get_action crashes on the terminal state). + if self.env.reward[r][c] > 0: + break + a = picker(list(self.agent_pos)) + if a is None or a == [] or a == 0.0: break + if isinstance(a, list): a = a[0] + dx, dy = DP_ACTIONS[a] + self.agent_pos = [max(0, min(WIDTH - 1, self.agent_pos[0] + dx)), + max(0, min(HEIGHT - 1, self.agent_pos[1] + dy))] + + def mainloop(self): + self._screen = _open(self.title, (WIDTH * UNIT, HEIGHT * UNIT + self.BAR)) + self._font = pygame.font.SysFont(None, 22) + self._small = pygame.font.SysFont(None, 16) + while True: + for e in pygame.event.get(): + if e.type == pygame.QUIT: + pygame.quit(); return + if e.type == pygame.MOUSEBUTTONDOWN and e.button == 1: + for rect, btn in zip(self._btn_rects(), self.buttons): + if rect.collidepoint(e.pos) and self._btn_enabled(btn): + label = btn[0] + self._press_label = label + self._press_frames = 6 + self._render() # draw the pressed state immediately + btn[1]() + self.clicks[label] = self.clicks.get(label, 0) + 1 + break + self._render() + time.sleep(0.03) + + @staticmethod + def _btn_enabled(btn): + # Button tuple is (label, handler) or (label, handler, enabled_fn). + return btn[2]() if len(btn) >= 3 else True + + def _btn_rects(self): + n = max(len(self.buttons), 1) + w = (WIDTH * UNIT) // n + return [pygame.Rect(i * w + 8, HEIGHT * UNIT + 8, w - 16, self.BAR - 16) + for i in range(len(self.buttons))] + + def _render(self): + s = self._screen + s.fill(WHITE) + _grid_lines(s, UNIT) + # PolicyEnv state is [row, col] but our draw helpers take [col, row] — swap. + _circle(s, 2, 2, UNIT, GOAL_COLOR) + for row, col in [(1, 2), (2, 1)]: + _triangle(s, col, row, UNIT, OBSTACLE_COLOR) + label = self._small.render("R : -1.0", True, TEXT_COLOR) + s.blit(label, (col * UNIT + 6, row * UNIT + 4)) + s.blit(self._small.render("R : +1.0", True, TEXT_COLOR), (2 * UNIT + 6, 2 * UNIT + 4)) + + # Agent first (filled), so V text and arrows render on top of it. + r, c = self.agent_pos + sz = int(UNIT * 0.55) + cx, cy = c * UNIT + UNIT // 2, r * UNIT + UNIT // 2 + pygame.draw.rect(s, AGENT_COLOR, pygame.Rect(cx - sz // 2, cy - sz // 2, sz, sz)) + + if self._values is not None: + for r in range(HEIGHT): + for c in range(WIDTH): + t = self._font.render(f"{self._values[r][c]:.2f}", True, TEXT_COLOR) + cx, cy = c * UNIT + UNIT // 2, r * UNIT + UNIT // 2 + s.blit(t, t.get_rect(center=(cx, cy + UNIT // 4))) + + if self._arrows is not None: + edge = [(0, -UNIT * 0.32), (0, UNIT * 0.32), + (-UNIT * 0.32, 0), (UNIT * 0.32, 0)] + for r in range(HEIGHT): + for c in range(WIDTH): + probs = self._arrows[r][c] + if not probs: continue + cx, cy = c * UNIT + UNIT // 2, r * UNIT + UNIT // 2 + for i, p in enumerate(probs): + if p > 0: + self._arrow(cx, cy, cx + edge[i][0], cy + edge[i][1]) + + for rect, btn in zip(self._btn_rects(), self.buttons): + enabled = self._btn_enabled(btn) + pressed = btn[0] == self._press_label and self._press_frames > 0 + if not enabled: + bg, fg, border = (245, 245, 245), (170, 170, 170), (200, 200, 200) + elif pressed: + bg, fg, border = (160, 180, 220), BLACK, BLACK # blue tint while held + else: + bg, fg, border = (220, 220, 220), BLACK, BLACK + pygame.draw.rect(s, bg, rect) + pygame.draw.rect(s, border, rect, 1) + t = self._font.render(btn[0], True, fg) + # Offset text slightly when pressed for a "depressed" feel. + center = (rect.centerx + (1 if pressed else 0), rect.centery + (1 if pressed else 0)) + s.blit(t, t.get_rect(center=center)) + if self._press_frames > 0: + self._press_frames -= 1 + + pygame.display.flip() + + def _arrow(self, x0, y0, x1, y1): + # Line from cell center to edge, then two short segments forming + # the arrowhead — each ~30 degrees off the backward direction. + pygame.draw.line(self._screen, BLACK, (x0, y0), (x1, y1), 2) + ang = math.atan2(y1 - y0, x1 - x0) + for sign in (-1, 1): + a = ang + sign * 0.5 + pygame.draw.line(self._screen, BLACK, (x1, y1), + (x1 - 8 * math.cos(a), y1 - 8 * math.sin(a)), 2) diff --git a/1-grid-world/gridworld_changing.png b/1-grid-world/gridworld_changing.png deleted file mode 100644 index 72bbb20c..00000000 Binary files a/1-grid-world/gridworld_changing.png and /dev/null differ diff --git a/1-grid-world/img/circle.png b/1-grid-world/img/circle.png deleted file mode 100644 index 7aeacd38..00000000 Binary files a/1-grid-world/img/circle.png and /dev/null differ diff --git a/1-grid-world/img/down.png b/1-grid-world/img/down.png deleted file mode 100644 index cd94e13f..00000000 Binary files a/1-grid-world/img/down.png and /dev/null differ diff --git a/1-grid-world/img/left.png b/1-grid-world/img/left.png deleted file mode 100644 index 079c57b6..00000000 Binary files a/1-grid-world/img/left.png and /dev/null differ diff --git a/1-grid-world/img/rectangle.png b/1-grid-world/img/rectangle.png deleted file mode 100644 index b7cea073..00000000 Binary files a/1-grid-world/img/rectangle.png and /dev/null differ diff --git a/1-grid-world/img/right.png b/1-grid-world/img/right.png deleted file mode 100644 index cbe1b1b5..00000000 Binary files a/1-grid-world/img/right.png and /dev/null differ diff --git a/1-grid-world/img/triangle.png b/1-grid-world/img/triangle.png deleted file mode 100644 index 1cd9db0a..00000000 Binary files a/1-grid-world/img/triangle.png and /dev/null differ diff --git a/1-grid-world/img/up.png b/1-grid-world/img/up.png deleted file mode 100644 index 6e9c2176..00000000 Binary files a/1-grid-world/img/up.png and /dev/null differ diff --git a/2-cartpole/1-dqn.py b/2-cartpole/1-dqn.py new file mode 100644 index 00000000..ddbb2198 --- /dev/null +++ b/2-cartpole/1-dqn.py @@ -0,0 +1,177 @@ +"""DQN agent for CartPole-v1. + +Mnih et al., 2015: "Human-level control through deep reinforcement +learning" (Nature). Key ingredients vs. plain online Q-learning: + + 1. Experience replay: store (s, a, r, s', done) and sample i.i.d. + minibatches, breaking correlation between consecutive samples. + 2. Target network: a periodically-copied snapshot of the Q-network used + to compute the TD target, which stabilizes bootstrapping. + +Off-policy Q-learning target (with target network Q_phi): + + y = r + gamma * max_{a'} Q_phi(s', a') if not done + y = r if done + +Loss (per minibatch sample): + + L(theta) = ( Q_theta(s)[a] - y )^2 +""" +import random +from collections import deque + +import numpy as np +import torch +import torch.nn as nn +import torch.optim as optim + +from env import make_env, parse_args, quit_if_window_closed, run_test_loop + +EPISODES = 300 +SAVE_PATH = "cartpole_dqn.pt" + + +# Approximator for Q(s, .). He-uniform init is friendly to ReLU. +class QNetwork(nn.Module): + def __init__(self, state_size, action_size): + super().__init__() + self.net = nn.Sequential( + nn.Linear(state_size, 24), + nn.ReLU(), + nn.Linear(24, 24), + nn.ReLU(), + nn.Linear(24, action_size), + ) + for m in self.net: + if isinstance(m, nn.Linear): + nn.init.kaiming_uniform_(m.weight, nonlinearity="relu") + nn.init.zeros_(m.bias) + + def forward(self, x): + return self.net(x) + + +class DQNAgent: + def __init__(self, state_size, action_size): + self.state_size = state_size + self.action_size = action_size + + # Hyperparameters. + self.discount_factor = 0.99 + self.learning_rate = 1e-3 + self.epsilon = 1.0 + self.epsilon_decay = 0.999 + self.epsilon_min = 0.01 + self.batch_size = 64 + # Wait until the replay buffer has enough samples before training. + self.train_start = 1000 + # Replay memory: a sliding window of recent transitions. + self.memory = deque(maxlen=2000) + + # Online network (trained) and target network (slow copy for bootstrapping). + self.model = QNetwork(state_size, action_size) + self.target_model = QNetwork(state_size, action_size) + self.update_target_model() + + self.optimizer = optim.Adam(self.model.parameters(), lr=self.learning_rate) + self.loss_fn = nn.MSELoss() + + # Hard update: target <- online. Called once per episode. + def update_target_model(self): + self.target_model.load_state_dict(self.model.state_dict()) + + # Epsilon-greedy over Q_theta(s, .). + def get_action(self, state): + if np.random.rand() <= self.epsilon: + return random.randrange(self.action_size) + with torch.no_grad(): + q = self.model(torch.as_tensor(state, dtype=torch.float32)) + return int(torch.argmax(q).item()) + + # Store transition and decay epsilon. + def append_sample(self, state, action, reward, next_state, done): + self.memory.append((state, action, reward, next_state, done)) + if self.epsilon > self.epsilon_min: + self.epsilon *= self.epsilon_decay + + # One SGD step on a uniformly-sampled minibatch from the replay buffer. + def train_model(self): + if len(self.memory) < self.train_start: + return + batch = random.sample(self.memory, self.batch_size) + states, actions, rewards, next_states, dones = zip(*batch) + + states = torch.as_tensor(np.array(states), dtype=torch.float32) + actions = torch.as_tensor(actions, dtype=torch.long) + rewards = torch.as_tensor(rewards, dtype=torch.float32) + next_states = torch.as_tensor(np.array(next_states), dtype=torch.float32) + dones = torch.as_tensor(dones, dtype=torch.float32) + + # Q_theta(s, a) + q_pred = self.model(states).gather(1, actions.unsqueeze(1)).squeeze(1) + # y = r + gamma * max_a' Q_phi(s', a') (zeroed out on terminal s') + with torch.no_grad(): + q_next = self.target_model(next_states).max(dim=1).values + target = rewards + (1.0 - dones) * self.discount_factor * q_next + + loss = self.loss_fn(q_pred, target) + self.optimizer.zero_grad() + loss.backward() + self.optimizer.step() + + +if __name__ == "__main__": + args = parse_args() + env = make_env(args) + state_size = env.observation_space.shape[0] + action_size = env.action_space.n + + agent = DQNAgent(state_size, action_size) + + if args.test: + agent.model.load_state_dict(torch.load(SAVE_PATH)) + agent.epsilon = 0.0 # fully greedy + run_test_loop(env, agent.get_action) + + scores = [] + solved = False + + for e in range(EPISODES): + if solved: + break + done = False + score = 0 + state, _ = env.reset() + state = np.array(state, dtype=np.float32) + + while not done: + quit_if_window_closed(env) + action = agent.get_action(state) + next_state, reward, terminated, truncated, _ = env.step(action) + done = terminated or truncated + next_state = np.array(next_state, dtype=np.float32) + score += reward # raw episode length so far + # Reward shaping (matches the rlcode-kr-v2 reference): a small + # +0.1 for each step that did not end the episode in failure, + # -1 for the failure step itself. Well-scaled magnitudes keep + # Q-values from exploding. + shaped_reward = 0.1 if not done or score == 500 else -1 + + agent.append_sample(state, action, shaped_reward, next_state, done) + # Train at every environment step. + agent.train_model() + state = next_state + + if done: + # Update target network once per episode. + agent.update_target_model() + scores.append(score) + print(f"episode: {e} score: {score} memory: {len(agent.memory)} epsilon: {agent.epsilon:.4f}") + + # Early stop when consistently near max episode length. + if np.mean(scores[-min(10, len(scores)):]) > 490: + solved = True + break + + torch.save(agent.model.state_dict(), SAVE_PATH) + print(f"Saved trained model to {SAVE_PATH}") diff --git a/2-cartpole/1-dqn/SumTree.py b/2-cartpole/1-dqn/SumTree.py deleted file mode 100644 index 1b72e9ea..00000000 --- a/2-cartpole/1-dqn/SumTree.py +++ /dev/null @@ -1,55 +0,0 @@ -import numpy - - -class SumTree: - write = 0 - - def __init__(self, capacity): - self.capacity = capacity - self.tree = numpy.zeros(2 * capacity - 1) - self.data = numpy.zeros(capacity, dtype=object) - - def _propagate(self, idx, change): - parent = (idx - 1) // 2 - - self.tree[parent] += change - - if parent != 0: - self._propagate(parent, change) - - def _retrieve(self, idx, s): - left = 2 * idx + 1 - right = left + 1 - - if left >= len(self.tree): - return idx - - if s <= self.tree[left]: - return self._retrieve(left, s) - else: - return self._retrieve(right, s - self.tree[left]) - - def total(self): - return self.tree[0] - - def add(self, p, data): - idx = self.write + self.capacity - 1 - - self.data[self.write] = data - self.update(idx, p) - - self.write += 1 - if self.write >= self.capacity: - self.write = 0 - - def update(self, idx, p): - change = p - self.tree[idx] - - self.tree[idx] = p - self._propagate(idx, change) - - def get(self, s): - idx = self._retrieve(0, s) - dataIdx = idx - self.capacity + 1 - - return (idx, self.tree[idx], self.data[dataIdx]) diff --git a/2-cartpole/1-dqn/cartpole_dqn.py b/2-cartpole/1-dqn/cartpole_dqn.py deleted file mode 100644 index 8b2baaf0..00000000 --- a/2-cartpole/1-dqn/cartpole_dqn.py +++ /dev/null @@ -1,169 +0,0 @@ -import sys -import gym -import pylab -import random -import numpy as np -from collections import deque -from keras.layers import Dense -from keras.optimizers import Adam -from keras.models import Sequential - -EPISODES = 300 - - -# DQN Agent for the Cartpole -# it uses Neural Network to approximate q function -# and replay memory & target q network -class DQNAgent: - def __init__(self, state_size, action_size): - # if you want to see Cartpole learning, then change to True - self.render = False - self.load_model = False - - # get size of state and action - self.state_size = state_size - self.action_size = action_size - - # These are hyper parameters for the DQN - self.discount_factor = 0.99 - self.learning_rate = 0.001 - self.epsilon = 1.0 - self.epsilon_decay = 0.999 - self.epsilon_min = 0.01 - self.batch_size = 64 - self.train_start = 1000 - # create replay memory using deque - self.memory = deque(maxlen=2000) - - # create main model and target model - self.model = self.build_model() - self.target_model = self.build_model() - - # initialize target model - self.update_target_model() - - if self.load_model: - self.model.load_weights("./save_model/cartpole_dqn.h5") - - # approximate Q function using Neural Network - # state is input and Q Value of each action is output of network - def build_model(self): - model = Sequential() - model.add(Dense(24, input_dim=self.state_size, activation='relu', - kernel_initializer='he_uniform')) - model.add(Dense(24, activation='relu', - kernel_initializer='he_uniform')) - model.add(Dense(self.action_size, activation='linear', - kernel_initializer='he_uniform')) - model.summary() - model.compile(loss='mse', optimizer=Adam(lr=self.learning_rate)) - return model - - # after some time interval update the target model to be same with model - def update_target_model(self): - self.target_model.set_weights(self.model.get_weights()) - - # get action from model using epsilon-greedy policy - def get_action(self, state): - if np.random.rand() <= self.epsilon: - return random.randrange(self.action_size) - else: - q_value = self.model.predict(state) - return np.argmax(q_value[0]) - - # save sample to the replay memory - def append_sample(self, state, action, reward, next_state, done): - self.memory.append((state, action, reward, next_state, done)) - if self.epsilon > self.epsilon_min: - self.epsilon *= self.epsilon_decay - - # pick samples randomly from replay memory (with batch_size) - def train_model(self): - if len(self.memory) < self.train_start: - return - batch_size = min(self.batch_size, len(self.memory)) - mini_batch = random.sample(self.memory, batch_size) - - update_input = np.zeros((batch_size, self.state_size)) - update_target = np.zeros((batch_size, self.state_size)) - action, reward, done = [], [], [] - - for i in range(self.batch_size): - update_input[i] = mini_batch[i][0] - action.append(mini_batch[i][1]) - reward.append(mini_batch[i][2]) - update_target[i] = mini_batch[i][3] - done.append(mini_batch[i][4]) - - target = self.model.predict(update_input) - target_val = self.target_model.predict(update_target) - - for i in range(self.batch_size): - # Q Learning: get maximum Q value at s' from target model - if done[i]: - target[i][action[i]] = reward[i] - else: - target[i][action[i]] = reward[i] + self.discount_factor * ( - np.amax(target_val[i])) - - # and do the model fit! - self.model.fit(update_input, target, batch_size=self.batch_size, - epochs=1, verbose=0) - - -if __name__ == "__main__": - # In case of CartPole-v1, maximum length of episode is 500 - env = gym.make('CartPole-v1') - # get size of state and action from environment - state_size = env.observation_space.shape[0] - action_size = env.action_space.n - - agent = DQNAgent(state_size, action_size) - - scores, episodes = [], [] - - for e in range(EPISODES): - done = False - score = 0 - state = env.reset() - state = np.reshape(state, [1, state_size]) - - while not done: - if agent.render: - env.render() - - # get action for the current state and go one step in environment - action = agent.get_action(state) - next_state, reward, done, info = env.step(action) - next_state = np.reshape(next_state, [1, state_size]) - # if an action make the episode end, then gives penalty of -100 - reward = reward if not done or score == 499 else -100 - - # save the sample to the replay memory - agent.append_sample(state, action, reward, next_state, done) - # every time step do the training - agent.train_model() - score += reward - state = next_state - - if done: - # every episode update the target model to be same with model - agent.update_target_model() - - # every episode, plot the play time - score = score if score == 500 else score + 100 - scores.append(score) - episodes.append(e) - pylab.plot(episodes, scores, 'b') - pylab.savefig("./save_graph/cartpole_dqn.png") - print("episode:", e, " score:", score, " memory length:", - len(agent.memory), " epsilon:", agent.epsilon) - - # if the mean of scores of last 10 episode is bigger than 490 - # stop training - if np.mean(scores[-min(10, len(scores)):]) > 490: - sys.exit() - - # save the model - if e % 50 == 0: - agent.model.save_weights("./save_model/cartpole_dqn.h5") diff --git a/2-cartpole/1-dqn/cartpole_only_per.py b/2-cartpole/1-dqn/cartpole_only_per.py deleted file mode 100644 index 1a66d86b..00000000 --- a/2-cartpole/1-dqn/cartpole_only_per.py +++ /dev/null @@ -1,224 +0,0 @@ -import sys -import gym -import pylab -import random -import numpy as np -from SumTree import SumTree -from collections import deque -from keras.layers import Dense -from keras.optimizers import Adam -from keras.models import Sequential - -EPISODES = 300 - - -# 카트폴 예제에서의 DQN 에이전트 -class DQNAgent: - def __init__(self, state_size, action_size): - self.render = False - self.load_model = False - - # 상태와 행동의 크기 정의 - self.state_size = state_size - self.action_size = action_size - - # DQN 하이퍼파라미터 - self.discount_factor = 0.99 - self.learning_rate = 0.001 - self.epsilon = 1.0 - self.epsilon_decay = 0.999 - self.epsilon_min = 0.01 - self.batch_size = 64 - self.train_start = 2000 - self.memory_size = 2000 - - # 리플레이 메모리, 최대 크기 2000 - self.memory = Memory(self.memory_size) - - # 모델과 타깃 모델 생성 - self.model = self.build_model() - self.target_model = self.build_model() - - # 타깃 모델 초기화 - self.update_target_model() - - if self.load_model: - self.model.load_weights("./save_model/cartpole_dqn_trained.h5") - - # 상태가 입력, 큐함수가 출력인 인공신경망 생성 - def build_model(self): - model = Sequential() - model.add(Dense(24, input_dim=self.state_size, activation='relu', - kernel_initializer='he_uniform')) - model.add(Dense(24, activation='relu', - kernel_initializer='he_uniform')) - model.add(Dense(self.action_size, activation='linear', - kernel_initializer='he_uniform')) - model.summary() - model.compile(loss='mse', optimizer=Adam(lr=self.learning_rate)) - return model - - # 타깃 모델을 모델의 가중치로 업데이트 - def update_target_model(self): - self.target_model.set_weights(self.model.get_weights()) - - # 입실론 탐욕 정책으로 행동 선택 - def get_action(self, state): - if np.random.rand() <= self.epsilon: - return random.randrange(self.action_size) - else: - q_value = self.model.predict(state) - return np.argmax(q_value[0]) - - # 샘플 을 리플레이 메모리에 저장 - def append_sample(self, state, action, reward, next_state, done): - if self.epsilon == 1: - done = True - - # TD-error 를 구해서 같이 메모리에 저장 - target = self.model.predict([state]) - old_val = target[0][action] - target_val = self.target_model.predict([next_state]) - if done: - target[0][action] = reward - else: - target[0][action] = reward + self.discount_factor * ( - np.amax(target_val[0])) - error = abs(old_val - target[0][action]) - - self.memory.add(error, (state, action, reward, next_state, done)) - - # 리플레이 메모리에서 무작위로 추출한 배치로 모델 학습 - def train_model(self): - if self.epsilon > self.epsilon_min: - self.epsilon *= self.epsilon_decay - - # 메모리에서 배치 크기만큼 무작위로 샘플 추출 - mini_batch = self.memory.sample(self.batch_size) - - errors = np.zeros(self.batch_size) - states = np.zeros((self.batch_size, self.state_size)) - next_states = np.zeros((self.batch_size, self.state_size)) - actions, rewards, dones = [], [], [] - - for i in range(self.batch_size): - states[i] = mini_batch[i][1][0] - actions.append(mini_batch[i][1][1]) - rewards.append(mini_batch[i][1][2]) - next_states[i] = mini_batch[i][1][3] - dones.append(mini_batch[i][1][4]) - - # 현재 상태에 대한 모델의 큐함수 - # 다음 상태에 대한 타깃 모델의 큐함수 - target = self.model.predict(states) - target_val = self.target_model.predict(next_states) - - # 벨만 최적 방정식을 이용한 업데이트 타깃 - for i in range(self.batch_size): - old_val = target[i][actions[i]] - if dones[i]: - target[i][actions[i]] = rewards[i] - else: - target[i][actions[i]] = rewards[i] + self.discount_factor * ( - np.amax(target_val[i])) - # TD-error를 저장 - errors[i] = abs(old_val - target[i][actions[i]]) - - # TD-error로 priority 업데이트 - for i in range(self.batch_size): - idx = mini_batch[i][0] - self.memory.update(idx, errors[i]) - - self.model.fit(states, target, batch_size=self.batch_size, - epochs=1, verbose=0) - - -class Memory: # stored as ( s, a, r, s_ ) in SumTree - e = 0.01 - a = 0.6 - - def __init__(self, capacity): - self.tree = SumTree(capacity) - - def _getPriority(self, error): - return (error + self.e) ** self.a - - def add(self, error, sample): - p = self._getPriority(error) - self.tree.add(p, sample) - - def sample(self, n): - batch = [] - segment = self.tree.total() / n - - for i in range(n): - a = segment * i - b = segment * (i + 1) - - s = random.uniform(a, b) - (idx, p, data) = self.tree.get(s) - batch.append((idx, data)) - - return batch - - def update(self, idx, error): - p = self._getPriority(error) - self.tree.update(idx, p) - - -if __name__ == "__main__": - # CartPole-v1 환경, 최대 타임스텝 수가 500 - env = gym.make('CartPole-v1') - state_size = env.observation_space.shape[0] - action_size = env.action_space.n - - # DQN 에이전트 생성 - agent = DQNAgent(state_size, action_size) - - scores, episodes = [], [] - - step = 0 - for e in range(EPISODES): - done = False - score = 0 - # env 초기화 - state = env.reset() - state = np.reshape(state, [1, state_size]) - - while not done: - if agent.render: - env.render() - step += 1 - # 현재 상태로 행동을 선택 - action = agent.get_action(state) - # 선택한 행동으로 환경에서 한 타임스텝 진행 - next_state, reward, done, info = env.step(action) - next_state = np.reshape(next_state, [1, state_size]) - # 에피소드가 중간에 끝나면 -100 보상 - r = reward if not done or score+reward == 500 else -10 - # 리플레이 메모리에 샘플 저장 - agent.append_sample(state, action, r, next_state, done) - # 매 타임스텝마다 학습 - if step >= agent.train_start: - agent.train_model() - - score += reward - state = next_state - - if done: - # 각 에피소드마다 타깃 모델을 모델의 가중치로 업데이트 - agent.update_target_model() - -# score = score if score == 500 else score + 100 - # 에피소드마다 학습 결과 출력 - scores.append(score) - episodes.append(e) - pylab.plot(episodes, scores, 'b') - pylab.savefig("./save_graph/cartpole_dqn.png") - print("episode:", e, " score:", score, " memory length:", - step if step <= agent.memory_size else agent.memory_size, " epsilon:", agent.epsilon) - - # 이전 10개 에피소드의 점수 평균이 490보다 크면 학습 중단 - if np.mean(scores[-min(10, len(scores)):]) > 490: - agent.model.save_weights("./save_model/cartpole_dqn.h5") - sys.exit() diff --git a/2-cartpole/1-dqn/save_graph/Cartpole_DQN.png b/2-cartpole/1-dqn/save_graph/Cartpole_DQN.png deleted file mode 100644 index 49114fd6..00000000 Binary files a/2-cartpole/1-dqn/save_graph/Cartpole_DQN.png and /dev/null differ diff --git a/2-cartpole/1-dqn/save_model/cartpole_dqn.h5 b/2-cartpole/1-dqn/save_model/cartpole_dqn.h5 deleted file mode 100644 index ba846f85..00000000 Binary files a/2-cartpole/1-dqn/save_model/cartpole_dqn.h5 and /dev/null differ diff --git a/2-cartpole/2-a2c.py b/2-cartpole/2-a2c.py new file mode 100644 index 00000000..a7d145a7 --- /dev/null +++ b/2-cartpole/2-a2c.py @@ -0,0 +1,159 @@ +"""A2C (Advantage Actor-Critic) agent for CartPole-v1. + +Mnih et al., 2016: "Asynchronous Methods for Deep Reinforcement Learning" +(A3C paper; A2C is the synchronous variant). + +Two networks: + - Actor pi_theta(a|s): policy over actions. + - Critic V_w(s): state-value baseline. + +One-step TD advantage: + + A(s, a) = r + gamma * V_w(s') - V_w(s) (= 0 on terminal s') + +Updates (one-step, online — like a TD(0) actor-critic): + + Actor: maximize log pi_theta(a|s) * A(s, a) (A is treated as constant) + Critic: minimize ( V_w(s) - (r + gamma * V_w(s')) )^2 + +Subtracting V_w(s) is the variance-reduction baseline; using a learned V +(rather than the Monte-Carlo return) is what makes this *actor-critic*. +""" + +import numpy as np +import torch +import torch.nn as nn +import torch.optim as optim + +from env import make_env, parse_args, quit_if_window_closed, run_test_loop + +EPISODES = 1000 +SAVE_PATH = "cartpole_a2c.pt" + + +# Policy network: outputs logits over actions. +class Actor(nn.Module): + def __init__(self, state_size, action_size): + super().__init__() + self.fc1 = nn.Linear(state_size, 24) + self.fc2 = nn.Linear(24, action_size) + nn.init.kaiming_uniform_(self.fc1.weight, nonlinearity="relu") + nn.init.kaiming_uniform_(self.fc2.weight, nonlinearity="relu") + + def forward(self, x): + return self.fc2(torch.relu(self.fc1(x))) + + +# Value network: outputs a scalar V(s). +class Critic(nn.Module): + def __init__(self, state_size): + super().__init__() + self.fc1 = nn.Linear(state_size, 24) + self.fc2 = nn.Linear(24, 1) + nn.init.kaiming_uniform_(self.fc1.weight, nonlinearity="relu") + nn.init.kaiming_uniform_(self.fc2.weight, nonlinearity="relu") + + def forward(self, x): + return self.fc2(torch.relu(self.fc1(x))).squeeze(-1) + + +class A2CAgent: + def __init__(self, state_size, action_size): + self.state_size = state_size + self.action_size = action_size + self.discount_factor = 0.99 + # The critic typically uses a larger lr than the actor to keep the + # baseline tracking the policy. + self.actor_lr = 1e-3 + self.critic_lr = 5e-3 + + self.actor = Actor(state_size, action_size) + self.critic = Critic(state_size) + self.actor_opt = optim.Adam(self.actor.parameters(), lr=self.actor_lr) + self.critic_opt = optim.Adam(self.critic.parameters(), lr=self.critic_lr) + + # Sample a ~ pi_theta(.|s). + def get_action(self, state): + with torch.no_grad(): + logits = self.actor(torch.as_tensor(state, dtype=torch.float32)) + probs = torch.softmax(logits, dim=-1).numpy() + return int(np.random.choice(self.action_size, p=probs)) + + # One-step online update. + def train_model(self, state, action, reward, next_state, done): + state_t = torch.as_tensor(state, dtype=torch.float32) + next_state_t = torch.as_tensor(next_state, dtype=torch.float32) + + value = self.critic(state_t) + # TD target for the critic; treated as a constant for the gradient. + with torch.no_grad(): + next_value = self.critic(next_state_t) + target = torch.tensor(float(reward)) if done else reward + self.discount_factor * next_value + # Advantage: A(s,a) = target - V(s). Detach: the actor sees A as fixed. + advantage = (target - value).detach() + + # Actor loss: -log pi(a|s) * A (gradient ascent on log pi * A). + logits = self.actor(state_t) + log_probs = torch.log_softmax(logits, dim=-1) + actor_loss = -log_probs[action] * advantage + + # Critic loss: squared TD error. + critic_loss = (value - target).pow(2) + + self.actor_opt.zero_grad() + actor_loss.backward() + self.actor_opt.step() + + self.critic_opt.zero_grad() + critic_loss.backward() + self.critic_opt.step() + + +if __name__ == "__main__": + args = parse_args() + env = make_env(args) + state_size = env.observation_space.shape[0] + action_size = env.action_space.n + + agent = A2CAgent(state_size, action_size) + + if args.test: + ckpt = torch.load(SAVE_PATH) + agent.actor.load_state_dict(ckpt["actor"]) + agent.critic.load_state_dict(ckpt["critic"]) + run_test_loop(env, agent.get_action) + + scores = [] + solved = False + + for e in range(EPISODES): + if solved: + break + done = False + score = 0 + state, _ = env.reset() + state = np.array(state, dtype=np.float32) + + while not done: + quit_if_window_closed(env) + action = agent.get_action(state) + next_state, reward, terminated, truncated, _ = env.step(action) + done = terminated or truncated + next_state = np.array(next_state, dtype=np.float32) + score += reward # raw episode length + # Reward shaping (matches the rlcode-kr-v2 reference and DQN). + shaped_reward = 0.1 if not done or score == 500 else -1 + + agent.train_model(state, action, shaped_reward, next_state, done) + state = next_state + + if done: + scores.append(score) + print(f"episode: {e} score: {score}") + if np.mean(scores[-min(10, len(scores)):]) > 490: + solved = True + break + + torch.save({"actor": agent.actor.state_dict(), + "critic": agent.critic.state_dict()}, SAVE_PATH) + print(f"Saved trained model to {SAVE_PATH}") diff --git a/2-cartpole/2-double-dqn/cartpole_ddqn.py b/2-cartpole/2-double-dqn/cartpole_ddqn.py deleted file mode 100644 index 73c51140..00000000 --- a/2-cartpole/2-double-dqn/cartpole_ddqn.py +++ /dev/null @@ -1,175 +0,0 @@ -import sys -import gym -import pylab -import random -import numpy as np -from collections import deque -from keras.layers import Dense -from keras.optimizers import Adam -from keras.models import Sequential - -EPISODES = 300 - - -# Double DQN Agent for the Cartpole -# it uses Neural Network to approximate q function -# and replay memory & target q network -class DoubleDQNAgent: - def __init__(self, state_size, action_size): - # if you want to see Cartpole learning, then change to True - self.render = False - self.load_model = False - # get size of state and action - self.state_size = state_size - self.action_size = action_size - - # these is hyper parameters for the Double DQN - self.discount_factor = 0.99 - self.learning_rate = 0.001 - self.epsilon = 1.0 - self.epsilon_decay = 0.999 - self.epsilon_min = 0.01 - self.batch_size = 64 - self.train_start = 1000 - # create replay memory using deque - self.memory = deque(maxlen=2000) - - # create main model and target model - self.model = self.build_model() - self.target_model = self.build_model() - - # initialize target model - self.update_target_model() - - if self.load_model: - self.model.load_weights("./save_model/cartpole_ddqn.h5") - - # approximate Q function using Neural Network - # state is input and Q Value of each action is output of network - def build_model(self): - model = Sequential() - model.add(Dense(24, input_dim=self.state_size, activation='relu', - kernel_initializer='he_uniform')) - model.add(Dense(24, activation='relu', - kernel_initializer='he_uniform')) - model.add(Dense(self.action_size, activation='linear', - kernel_initializer='he_uniform')) - model.summary() - model.compile(loss='mse', optimizer=Adam(lr=self.learning_rate)) - return model - - # after some time interval update the target model to be same with model - def update_target_model(self): - self.target_model.set_weights(self.model.get_weights()) - - # get action from model using epsilon-greedy policy - def get_action(self, state): - if np.random.rand() <= self.epsilon: - return random.randrange(self.action_size) - else: - q_value = self.model.predict(state) - return np.argmax(q_value[0]) - - # save sample to the replay memory - def append_sample(self, state, action, reward, next_state, done): - self.memory.append((state, action, reward, next_state, done)) - if self.epsilon > self.epsilon_min: - self.epsilon *= self.epsilon_decay - - # pick samples randomly from replay memory (with batch_size) - def train_model(self): - if len(self.memory) < self.train_start: - return - batch_size = min(self.batch_size, len(self.memory)) - mini_batch = random.sample(self.memory, batch_size) - - update_input = np.zeros((batch_size, self.state_size)) - update_target = np.zeros((batch_size, self.state_size)) - action, reward, done = [], [], [] - - for i in range(batch_size): - update_input[i] = mini_batch[i][0] - action.append(mini_batch[i][1]) - reward.append(mini_batch[i][2]) - update_target[i] = mini_batch[i][3] - done.append(mini_batch[i][4]) - - target = self.model.predict(update_input) - target_next = self.model.predict(update_target) - target_val = self.target_model.predict(update_target) - - for i in range(self.batch_size): - # like Q Learning, get maximum Q value at s' - # But from target model - if done[i]: - target[i][action[i]] = reward[i] - else: - # the key point of Double DQN - # selection of action is from model - # update is from target model - a = np.argmax(target_next[i]) - target[i][action[i]] = reward[i] + self.discount_factor * ( - target_val[i][a]) - - # make minibatch which includes target q value and predicted q value - # and do the model fit! - self.model.fit(update_input, target, batch_size=self.batch_size, - epochs=1, verbose=0) - - -if __name__ == "__main__": - # In case of CartPole-v1, you can play until 500 time step - env = gym.make('CartPole-v1') - # get size of state and action from environment - state_size = env.observation_space.shape[0] - action_size = env.action_space.n - - agent = DoubleDQNAgent(state_size, action_size) - - scores, episodes = [], [] - - for e in range(EPISODES): - done = False - score = 0 - state = env.reset() - state = np.reshape(state, [1, state_size]) - - while not done: - if agent.render: - env.render() - - # get action for the current state and go one step in environment - action = agent.get_action(state) - next_state, reward, done, info = env.step(action) - next_state = np.reshape(next_state, [1, state_size]) - # if an action make the episode end, then gives penalty of -100 - reward = reward if not done or score == 499 else -100 - - # save the sample to the replay memory - agent.append_sample(state, action, reward, next_state, done) - # every time step do the training - agent.train_model() - score += reward - state = next_state - - if done: - # every episode update the target model to be same with model - agent.update_target_model() - - # every episode, plot the play time - score = score if score == 500 else score + 100 - scores.append(score) - episodes.append(e) - pylab.plot(episodes, scores, 'b') - pylab.savefig("./save_graph/cartpole_ddqn.png") - print("episode:", e, " score:", score, " memory length:", - len(agent.memory), " epsilon:", agent.epsilon) - - # if the mean of scores of last 10 episode is bigger than 490 - # stop training - if np.mean(scores[-min(10, len(scores)):]) > 490: - sys.exit() - - # save the model - if e % 50 == 0: - agent.model.save_weights("./save_model/cartpole_ddqn.h5") diff --git a/2-cartpole/2-double-dqn/save_graph/cartpole_ddqn.png b/2-cartpole/2-double-dqn/save_graph/cartpole_ddqn.png deleted file mode 100644 index 26c4fed0..00000000 Binary files a/2-cartpole/2-double-dqn/save_graph/cartpole_ddqn.png and /dev/null differ diff --git a/2-cartpole/2-double-dqn/save_model/cartpole_ddqn.h5 b/2-cartpole/2-double-dqn/save_model/cartpole_ddqn.h5 deleted file mode 100644 index c54c9886..00000000 Binary files a/2-cartpole/2-double-dqn/save_model/cartpole_ddqn.h5 and /dev/null differ diff --git a/2-cartpole/3-ppo.py b/2-cartpole/3-ppo.py new file mode 100644 index 00000000..174d2665 --- /dev/null +++ b/2-cartpole/3-ppo.py @@ -0,0 +1,216 @@ +"""PPO (Proximal Policy Optimization) agent for CartPole-v1. + +Schulman et al., 2017: "Proximal Policy Optimization Algorithms" +(arXiv:1707.06347). Also uses GAE from Schulman et al., 2016: +"High-Dimensional Continuous Control Using Generalized Advantage +Estimation" (arXiv:1506.02438). + +PPO is an on-policy actor-critic method. Define the probability ratio: + + r_t(theta) = pi_theta(a_t | s_t) / pi_theta_old(a_t | s_t) + +Clipped surrogate objective (the heart of PPO): + + L^CLIP(theta) = E_t [ min( r_t(theta) * A_t, + clip(r_t(theta), 1 - eps, 1 + eps) * A_t ) ] + +By clipping the ratio we discourage updates that move pi too far from +pi_old in a single step — this is what lets us reuse a batch of data +for several gradient epochs while staying near the trust region. + +Generalized Advantage Estimation (GAE-lambda): + + delta_t = r_t + gamma * V(s_{t+1}) * (1 - done_t) - V(s_t) + A_t = delta_t + (gamma * lambda) * (1 - done_t) * A_{t+1} + +Total loss combines clipped policy loss, value MSE, and an entropy bonus: + + L = L^CLIP - c_v * MSE(V, returns) + c_e * H[pi] +""" + +import numpy as np +import torch +import torch.nn as nn +import torch.optim as optim + +from env import make_env, parse_args, quit_if_window_closed, run_test_loop + +EPISODES = 1500 +SAVE_PATH = "cartpole_ppo.pt" +# Steps collected per update; PPO is batch-based, not single-step like A2C. +# 256 is too small for a single env on CartPole — GAE gets noisy and PPO +# oscillates. 1024 (with 4 epochs / 64 minibatches) is closer to the +# CleanRL single-env reference and gives much steadier learning. +ROLLOUT_STEPS = 1024 +# Number of times we sweep over the collected batch each update. +EPOCHS = 4 +MINIBATCH_SIZE = 64 +# Clip range epsilon from the PPO paper; 0.2 is the canonical value. +CLIP_COEF = 0.2 +GAMMA = 0.99 +GAE_LAMBDA = 0.95 +LR = 3e-4 +# Value-loss weight and entropy bonus weight. +VALUE_COEF = 0.5 +ENTROPY_COEF = 0.01 + + +def _ortho(layer, gain): + """Orthogonal init — a standard PPO stability trick (CleanRL-style).""" + nn.init.orthogonal_(layer.weight, gain) + nn.init.zeros_(layer.bias) + return layer + + +# Shared-trunk actor-critic: two-layer MLP with tanh, then policy and value heads. +class ActorCritic(nn.Module): + def __init__(self, state_size, action_size): + super().__init__() + # gain = sqrt(2) for the tanh trunk, 0.01 for the policy head + # (keeps initial action distribution close to uniform), 1 for the + # value head. These are the standard PPO-paper / CleanRL choices. + self.shared = nn.Sequential( + _ortho(nn.Linear(state_size, 64), gain=2 ** 0.5), + nn.Tanh(), + _ortho(nn.Linear(64, 64), gain=2 ** 0.5), + nn.Tanh(), + ) + self.policy = _ortho(nn.Linear(64, action_size), gain=0.01) + self.value = _ortho(nn.Linear(64, 1), gain=1.0) + + def forward(self, x): + h = self.shared(x) + return self.policy(h), self.value(h).squeeze(-1) + + +# GAE-lambda: backward recursion over the collected rollout. +# `dones` marks terminal transitions; the recursion is reset there. +def compute_gae(rewards, values, dones, last_value): + advantages = np.zeros_like(rewards, dtype=np.float32) + gae = 0.0 + for t in reversed(range(len(rewards))): + next_v = last_value if t == len(rewards) - 1 else values[t + 1] + next_nonterminal = 1.0 - dones[t] + # delta_t = r_t + gamma * V(s_{t+1}) - V(s_t) + delta = rewards[t] + GAMMA * next_v * next_nonterminal - values[t] + # A_t = delta_t + gamma * lambda * A_{t+1} + gae = delta + GAMMA * GAE_LAMBDA * next_nonterminal * gae + advantages[t] = gae + # Returns used as the value target: R_t = A_t + V(s_t). + returns = advantages + values + return advantages, returns + + +if __name__ == "__main__": + args = parse_args() + env = make_env(args) + state_size = env.observation_space.shape[0] + action_size = env.action_space.n + + model = ActorCritic(state_size, action_size) + optimizer = optim.Adam(model.parameters(), lr=LR) + + if args.test: + model.load_state_dict(torch.load(SAVE_PATH)) + + def pick(state): + with torch.no_grad(): + logits, _ = model(torch.as_tensor(state)) + return int(torch.distributions.Categorical(logits=logits).sample().item()) + + run_test_loop(env, pick) + + state, _ = env.reset() + state = np.array(state, dtype=np.float32) + ep_return = 0.0 + ep_returns = [] + + for episode in range(EPISODES): + # --- 1. Roll out the current policy for ROLLOUT_STEPS. --- + obs_buf = np.zeros((ROLLOUT_STEPS, state_size), dtype=np.float32) + act_buf = np.zeros(ROLLOUT_STEPS, dtype=np.int64) + logp_buf = np.zeros(ROLLOUT_STEPS, dtype=np.float32) + rew_buf = np.zeros(ROLLOUT_STEPS, dtype=np.float32) + done_buf = np.zeros(ROLLOUT_STEPS, dtype=np.float32) + val_buf = np.zeros(ROLLOUT_STEPS, dtype=np.float32) + + for t in range(ROLLOUT_STEPS): + quit_if_window_closed(env) + with torch.no_grad(): + logits, value = model(torch.as_tensor(state)) + # Categorical handles softmax + sampling + log_prob cleanly. + dist = torch.distributions.Categorical(logits=logits) + action = dist.sample() + logp = dist.log_prob(action) + + obs_buf[t] = state + act_buf[t] = action.item() + # Stash log pi_theta_old(a_t | s_t) for the ratio computation later. + logp_buf[t] = logp.item() + val_buf[t] = value.item() + + next_state, reward, terminated, truncated, _ = env.step(int(action.item())) + done = terminated or truncated + ep_return += reward # raw episode length (for reporting) + # Reward shaping (matches DQN / A2C / rlcode-kr-v2): +0.1 per + # surviving step, -1 on the failure step. Without this PPO + # gets a very weak signal on CartPole and oscillates. + rew_buf[t] = 0.1 if not done or ep_return == 500 else -1 + done_buf[t] = float(done) + + if done: + ep_returns.append(ep_return) + ep_return = 0.0 + next_state, _ = env.reset() + state = np.array(next_state, dtype=np.float32) + + # --- 2. Compute advantages and returns via GAE. --- + # Bootstrap with V(s_T) at the rollout boundary (not necessarily terminal). + with torch.no_grad(): + _, last_value = model(torch.as_tensor(state)) + advantages, returns = compute_gae(rew_buf, val_buf, done_buf, last_value.item()) + # Per-batch advantage normalization (standard PPO trick). + advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-8) + + obs_t = torch.as_tensor(obs_buf) + act_t = torch.as_tensor(act_buf) + old_logp_t = torch.as_tensor(logp_buf) + adv_t = torch.as_tensor(advantages) + ret_t = torch.as_tensor(returns) + + # --- 3. Multiple epochs of minibatch SGD on the clipped surrogate. --- + idx = np.arange(ROLLOUT_STEPS) + for _ in range(EPOCHS): + np.random.shuffle(idx) + for start in range(0, ROLLOUT_STEPS, MINIBATCH_SIZE): + mb = idx[start:start + MINIBATCH_SIZE] + logits, values = model(obs_t[mb]) + dist = torch.distributions.Categorical(logits=logits) + new_logp = dist.log_prob(act_t[mb]) + entropy = dist.entropy().mean() + + # ratio = pi_new / pi_old = exp(log pi_new - log pi_old) + ratio = (new_logp - old_logp_t[mb]).exp() + unclipped = ratio * adv_t[mb] + clipped = torch.clamp(ratio, 1 - CLIP_COEF, 1 + CLIP_COEF) * adv_t[mb] + # PPO objective is the *min* of clipped and unclipped — pessimistic + # bound that ignores improvements outside the trust region. + policy_loss = -torch.min(unclipped, clipped).mean() + value_loss = (values - ret_t[mb]).pow(2).mean() + # Entropy bonus encourages exploration. + loss = policy_loss + VALUE_COEF * value_loss - ENTROPY_COEF * entropy + + optimizer.zero_grad() + loss.backward() + # Global grad clipping is a standard stabilizer in PPO. + nn.utils.clip_grad_norm_(model.parameters(), 0.5) + optimizer.step() + + if ep_returns: + recent = ep_returns[-10:] + print(f"update: {episode} recent_mean_return: {np.mean(recent):.1f} episodes: {len(ep_returns)}") + if len(recent) >= 10 and np.mean(recent) > 490: + break + + torch.save(model.state_dict(), SAVE_PATH) + print(f"Saved trained model to {SAVE_PATH}") diff --git a/2-cartpole/3-reinforce/cartpole_reinforce.py b/2-cartpole/3-reinforce/cartpole_reinforce.py deleted file mode 100644 index 040234d1..00000000 --- a/2-cartpole/3-reinforce/cartpole_reinforce.py +++ /dev/null @@ -1,146 +0,0 @@ -import sys -import gym -import pylab -import numpy as np -from keras.layers import Dense -from keras.models import Sequential -from keras.optimizers import Adam - -EPISODES = 1000 - - -# This is Policy Gradient agent for the Cartpole -# In this example, we use REINFORCE algorithm which uses monte-carlo update rule -class REINFORCEAgent: - def __init__(self, state_size, action_size): - # if you want to see Cartpole learning, then change to True - self.render = False - self.load_model = False - # get size of state and action - self.state_size = state_size - self.action_size = action_size - - # These are hyper parameters for the Policy Gradient - self.discount_factor = 0.99 - self.learning_rate = 0.001 - self.hidden1, self.hidden2 = 24, 24 - - # create model for policy network - self.model = self.build_model() - - # lists for the states, actions and rewards - self.states, self.actions, self.rewards = [], [], [] - - if self.load_model: - self.model.load_weights("./save_model/cartpole_reinforce.h5") - - # approximate policy using Neural Network - # state is input and probability of each action is output of network - def build_model(self): - model = Sequential() - model.add(Dense(self.hidden1, input_dim=self.state_size, activation='relu', kernel_initializer='glorot_uniform')) - model.add(Dense(self.hidden2, activation='relu', kernel_initializer='glorot_uniform')) - model.add(Dense(self.action_size, activation='softmax', kernel_initializer='glorot_uniform')) - model.summary() - # Using categorical crossentropy as a loss is a trick to easily - # implement the policy gradient. Categorical cross entropy is defined - # H(p, q) = sum(p_i * log(q_i)). For the action taken, a, you set - # p_a = advantage. q_a is the output of the policy network, which is - # the probability of taking the action a, i.e. policy(s, a). - # All other p_i are zero, thus we have H(p, q) = A * log(policy(s, a)) - model.compile(loss="categorical_crossentropy", optimizer=Adam(lr=self.learning_rate)) - return model - - # using the output of policy network, pick action stochastically - def get_action(self, state): - policy = self.model.predict(state, batch_size=1).flatten() - return np.random.choice(self.action_size, 1, p=policy)[0] - - # In Policy Gradient, Q function is not available. - # Instead agent uses sample returns for evaluating policy - def discount_rewards(self, rewards): - discounted_rewards = np.zeros_like(rewards) - running_add = 0 - for t in reversed(range(0, len(rewards))): - running_add = running_add * self.discount_factor + rewards[t] - discounted_rewards[t] = running_add - return discounted_rewards - - # save of each step - def append_sample(self, state, action, reward): - self.states.append(state) - self.rewards.append(reward) - self.actions.append(action) - - # update policy network every episode - def train_model(self): - episode_length = len(self.states) - - discounted_rewards = self.discount_rewards(self.rewards) - discounted_rewards -= np.mean(discounted_rewards) - discounted_rewards /= np.std(discounted_rewards) - - update_inputs = np.zeros((episode_length, self.state_size)) - advantages = np.zeros((episode_length, self.action_size)) - - for i in range(episode_length): - update_inputs[i] = self.states[i] - advantages[i][self.actions[i]] = discounted_rewards[i] - - self.model.fit(update_inputs, advantages, epochs=1, verbose=0) - self.states, self.actions, self.rewards = [], [], [] - -if __name__ == "__main__": - # In case of CartPole-v1, you can play until 500 time step - env = gym.make('CartPole-v1') - # get size of state and action from environment - state_size = env.observation_space.shape[0] - action_size = env.action_space.n - - # make REINFORCE agent - agent = REINFORCEAgent(state_size, action_size) - - scores, episodes = [], [] - - for e in range(EPISODES): - done = False - score = 0 - state = env.reset() - state = np.reshape(state, [1, state_size]) - - while not done: - if agent.render: - env.render() - - # get action for the current state and go one step in environment - action = agent.get_action(state) - next_state, reward, done, info = env.step(action) - next_state = np.reshape(next_state, [1, state_size]) - reward = reward if not done or score == 499 else -100 - - # save the sample to the memory - agent.append_sample(state, action, reward) - - score += reward - state = next_state - - if done: - # every episode, agent learns from sample returns - agent.train_model() - - # every episode, plot the play time - score = score if score == 500 else score + 100 - scores.append(score) - episodes.append(e) - pylab.plot(episodes, scores, 'b') - pylab.savefig("./save_graph/cartpole_reinforce.png") - print("episode:", e, " score:", score) - - # if the mean of scores of last 10 episode is bigger than 490 - # stop training - if np.mean(scores[-min(10, len(scores)):]) > 490: - sys.exit() - - # save the model - if e % 50 == 0: - agent.model.save_weights("./save_model/cartpole_reinforce.h5") diff --git a/2-cartpole/3-reinforce/save_graph/cartpole_reinforce.png b/2-cartpole/3-reinforce/save_graph/cartpole_reinforce.png deleted file mode 100644 index dce280f2..00000000 Binary files a/2-cartpole/3-reinforce/save_graph/cartpole_reinforce.png and /dev/null differ diff --git a/2-cartpole/3-reinforce/save_model/cartpole_reinforce.h5 b/2-cartpole/3-reinforce/save_model/cartpole_reinforce.h5 deleted file mode 100644 index 18fb216b..00000000 Binary files a/2-cartpole/3-reinforce/save_model/cartpole_reinforce.h5 and /dev/null differ diff --git a/2-cartpole/4-actor-critic/cartpole_a2c.py b/2-cartpole/4-actor-critic/cartpole_a2c.py deleted file mode 100644 index fa6310a3..00000000 --- a/2-cartpole/4-actor-critic/cartpole_a2c.py +++ /dev/null @@ -1,135 +0,0 @@ -import sys -import gym -import pylab -import numpy as np -from keras.layers import Dense -from keras.models import Sequential -from keras.optimizers import Adam - -EPISODES = 1000 - - -# A2C(Advantage Actor-Critic) agent for the Cartpole -class A2CAgent: - def __init__(self, state_size, action_size): - # if you want to see Cartpole learning, then change to True - self.render = False - self.load_model = False - # get size of state and action - self.state_size = state_size - self.action_size = action_size - self.value_size = 1 - - # These are hyper parameters for the Policy Gradient - self.discount_factor = 0.99 - self.actor_lr = 0.001 - self.critic_lr = 0.005 - - # create model for policy network - self.actor = self.build_actor() - self.critic = self.build_critic() - - if self.load_model: - self.actor.load_weights("./save_model/cartpole_actor.h5") - self.critic.load_weights("./save_model/cartpole_critic.h5") - - # approximate policy and value using Neural Network - # actor: state is input and probability of each action is output of model - def build_actor(self): - actor = Sequential() - actor.add(Dense(24, input_dim=self.state_size, activation='relu', - kernel_initializer='he_uniform')) - actor.add(Dense(self.action_size, activation='softmax', - kernel_initializer='he_uniform')) - actor.summary() - # See note regarding crossentropy in cartpole_reinforce.py - actor.compile(loss='categorical_crossentropy', - optimizer=Adam(lr=self.actor_lr)) - return actor - - # critic: state is input and value of state is output of model - def build_critic(self): - critic = Sequential() - critic.add(Dense(24, input_dim=self.state_size, activation='relu', - kernel_initializer='he_uniform')) - critic.add(Dense(self.value_size, activation='linear', - kernel_initializer='he_uniform')) - critic.summary() - critic.compile(loss="mse", optimizer=Adam(lr=self.critic_lr)) - return critic - - # using the output of policy network, pick action stochastically - def get_action(self, state): - policy = self.actor.predict(state, batch_size=1).flatten() - return np.random.choice(self.action_size, 1, p=policy)[0] - - # update policy network every episode - def train_model(self, state, action, reward, next_state, done): - target = np.zeros((1, self.value_size)) - advantages = np.zeros((1, self.action_size)) - - value = self.critic.predict(state)[0] - next_value = self.critic.predict(next_state)[0] - - if done: - advantages[0][action] = reward - value - target[0][0] = reward - else: - advantages[0][action] = reward + self.discount_factor * (next_value) - value - target[0][0] = reward + self.discount_factor * next_value - - self.actor.fit(state, advantages, epochs=1, verbose=0) - self.critic.fit(state, target, epochs=1, verbose=0) - - -if __name__ == "__main__": - # In case of CartPole-v1, maximum length of episode is 500 - env = gym.make('CartPole-v1') - # get size of state and action from environment - state_size = env.observation_space.shape[0] - action_size = env.action_space.n - - # make A2C agent - agent = A2CAgent(state_size, action_size) - - scores, episodes = [], [] - - for e in range(EPISODES): - done = False - score = 0 - state = env.reset() - state = np.reshape(state, [1, state_size]) - - while not done: - if agent.render: - env.render() - - action = agent.get_action(state) - next_state, reward, done, info = env.step(action) - next_state = np.reshape(next_state, [1, state_size]) - # if an action make the episode end, then gives penalty of -100 - reward = reward if not done or score == 499 else -100 - - agent.train_model(state, action, reward, next_state, done) - - score += reward - state = next_state - - if done: - # every episode, plot the play time - score = score if score == 500.0 else score + 100 - scores.append(score) - episodes.append(e) - pylab.plot(episodes, scores, 'b') - pylab.savefig("./save_graph/cartpole_a2c.png") - print("episode:", e, " score:", score) - - # if the mean of scores of last 10 episode is bigger than 490 - # stop training - if np.mean(scores[-min(10, len(scores)):]) > 490: - sys.exit() - - # save the model - if e % 50 == 0: - agent.actor.save_weights("./save_model/cartpole_actor.h5") - agent.critic.save_weights("./save_model/cartpole_critic.h5") diff --git a/2-cartpole/4-actor-critic/save_graph/cartpole_a2c.png b/2-cartpole/4-actor-critic/save_graph/cartpole_a2c.png deleted file mode 100644 index aedc6c4c..00000000 Binary files a/2-cartpole/4-actor-critic/save_graph/cartpole_a2c.png and /dev/null differ diff --git a/2-cartpole/4-actor-critic/save_model/cartpole_actor.h5 b/2-cartpole/4-actor-critic/save_model/cartpole_actor.h5 deleted file mode 100644 index 38b40bba..00000000 Binary files a/2-cartpole/4-actor-critic/save_model/cartpole_actor.h5 and /dev/null differ diff --git a/2-cartpole/4-actor-critic/save_model/cartpole_critic.h5 b/2-cartpole/4-actor-critic/save_model/cartpole_critic.h5 deleted file mode 100644 index 4cea5ef1..00000000 Binary files a/2-cartpole/4-actor-critic/save_model/cartpole_critic.h5 and /dev/null differ diff --git a/2-cartpole/5-a3c/cartpole_a3c.py b/2-cartpole/5-a3c/cartpole_a3c.py deleted file mode 100644 index f2721849..00000000 --- a/2-cartpole/5-a3c/cartpole_a3c.py +++ /dev/null @@ -1,223 +0,0 @@ -import threading -import numpy as np -import tensorflow as tf -import pylab -import time -import gym -from keras.layers import Dense, Input -from keras.models import Model -from keras.optimizers import Adam -from keras import backend as K - - -# global variables for threading -episode = 0 -scores = [] - -EPISODES = 2000 - -# This is A3C(Asynchronous Advantage Actor Critic) agent(global) for the Cartpole -# In this example, we use A3C algorithm -class A3CAgent: - def __init__(self, state_size, action_size, env_name): - # get size of state and action - self.state_size = state_size - self.action_size = action_size - - # get gym environment name - self.env_name = env_name - - # these are hyper parameters for the A3C - self.actor_lr = 0.001 - self.critic_lr = 0.001 - self.discount_factor = .99 - self.hidden1, self.hidden2 = 24, 24 - self.threads = 8 - - # create model for actor and critic network - self.actor, self.critic = self.build_model() - - # method for training actor and critic network - self.optimizer = [self.actor_optimizer(), self.critic_optimizer()] - - self.sess = tf.InteractiveSession() - K.set_session(self.sess) - self.sess.run(tf.global_variables_initializer()) - - # approximate policy and value using Neural Network - # actor -> state is input and probability of each action is output of network - # critic -> state is input and value of state is output of network - # actor and critic network share first hidden layer - def build_model(self): - state = Input(batch_shape=(None, self.state_size)) - shared = Dense(self.hidden1, input_dim=self.state_size, activation='relu', kernel_initializer='glorot_uniform')(state) - - actor_hidden = Dense(self.hidden2, activation='relu', kernel_initializer='glorot_uniform')(shared) - action_prob = Dense(self.action_size, activation='softmax', kernel_initializer='glorot_uniform')(actor_hidden) - - value_hidden = Dense(self.hidden2, activation='relu', kernel_initializer='he_uniform')(shared) - state_value = Dense(1, activation='linear', kernel_initializer='he_uniform')(value_hidden) - - actor = Model(inputs=state, outputs=action_prob) - critic = Model(inputs=state, outputs=state_value) - - actor._make_predict_function() - critic._make_predict_function() - - actor.summary() - critic.summary() - - return actor, critic - - # make loss function for Policy Gradient - # [log(action probability) * advantages] will be input for the back prop - # we add entropy of action probability to loss - def actor_optimizer(self): - action = K.placeholder(shape=(None, self.action_size)) - advantages = K.placeholder(shape=(None, )) - - policy = self.actor.output - - good_prob = K.sum(action * policy, axis=1) - eligibility = K.log(good_prob + 1e-10) * K.stop_gradient(advantages) - loss = -K.sum(eligibility) - - entropy = K.sum(policy * K.log(policy + 1e-10), axis=1) - - actor_loss = loss + 0.01*entropy - - optimizer = Adam(lr=self.actor_lr) - updates = optimizer.get_updates(self.actor.trainable_weights, [], actor_loss) - train = K.function([self.actor.input, action, advantages], [], updates=updates) - return train - - # make loss function for Value approximation - def critic_optimizer(self): - discounted_reward = K.placeholder(shape=(None, )) - - value = self.critic.output - - loss = K.mean(K.square(discounted_reward - value)) - - optimizer = Adam(lr=self.critic_lr) - updates = optimizer.get_updates(self.critic.trainable_weights, [], loss) - train = K.function([self.critic.input, discounted_reward], [], updates=updates) - return train - - # make agents(local) and start training - def train(self): - # self.load_model('./save_model/cartpole_a3c.h5') - agents = [Agent(i, self.actor, self.critic, self.optimizer, self.env_name, self.discount_factor, - self.action_size, self.state_size) for i in range(self.threads)] - - for agent in agents: - agent.start() - - while True: - time.sleep(20) - - plot = scores[:] - pylab.plot(range(len(plot)), plot, 'b') - pylab.savefig("./save_graph/cartpole_a3c.png") - - self.save_model('./save_model/cartpole_a3c.h5') - - def save_model(self, name): - self.actor.save_weights(name + "_actor.h5") - self.critic.save_weights(name + "_critic.h5") - - def load_model(self, name): - self.actor.load_weights(name + "_actor.h5") - self.critic.load_weights(name + "_critic.h5") - -# This is Agent(local) class for threading -class Agent(threading.Thread): - def __init__(self, index, actor, critic, optimizer, env_name, discount_factor, action_size, state_size): - threading.Thread.__init__(self) - - self.states = [] - self.rewards = [] - self.actions = [] - - self.index = index - self.actor = actor - self.critic = critic - self.optimizer = optimizer - self.env_name = env_name - self.discount_factor = discount_factor - self.action_size = action_size - self.state_size = state_size - - # Thread interactive with environment - def run(self): - global episode - env = gym.make(self.env_name) - while episode < EPISODES: - state = env.reset() - score = 0 - while True: - action = self.get_action(state) - next_state, reward, done, _ = env.step(action) - score += reward - - self.memory(state, action, reward) - - state = next_state - - if done: - episode += 1 - print("episode: ", episode, "/ score : ", score) - scores.append(score) - self.train_episode(score != 500) - break - - # In Policy Gradient, Q function is not available. - # Instead agent uses sample returns for evaluating policy - def discount_rewards(self, rewards, done=True): - discounted_rewards = np.zeros_like(rewards) - running_add = 0 - if not done: - running_add = self.critic.predict(np.reshape(self.states[-1], (1, self.state_size)))[0] - for t in reversed(range(0, len(rewards))): - running_add = running_add * self.discount_factor + rewards[t] - discounted_rewards[t] = running_add - return discounted_rewards - - # save of each step - # this is used for calculating discounted rewards - def memory(self, state, action, reward): - self.states.append(state) - act = np.zeros(self.action_size) - act[action] = 1 - self.actions.append(act) - self.rewards.append(reward) - - # update policy network and value network every episode - def train_episode(self, done): - discounted_rewards = self.discount_rewards(self.rewards, done) - - values = self.critic.predict(np.array(self.states)) - values = np.reshape(values, len(values)) - - advantages = discounted_rewards - values - - self.optimizer[0]([self.states, self.actions, advantages]) - self.optimizer[1]([self.states, discounted_rewards]) - self.states, self.actions, self.rewards = [], [], [] - - def get_action(self, state): - policy = self.actor.predict(np.reshape(state, [1, self.state_size]))[0] - return np.random.choice(self.action_size, 1, p=policy)[0] - - -if __name__ == "__main__": - env_name = 'CartPole-v1' - env = gym.make(env_name) - - state_size = env.observation_space.shape[0] - action_size = env.action_space.n - - env.close() - - global_agent = A3CAgent(state_size, action_size, env_name) - global_agent.train() diff --git a/2-cartpole/5-a3c/save_model/Cartpole_A3C_actor.h5 b/2-cartpole/5-a3c/save_model/Cartpole_A3C_actor.h5 deleted file mode 100644 index 33ab03a5..00000000 Binary files a/2-cartpole/5-a3c/save_model/Cartpole_A3C_actor.h5 and /dev/null differ diff --git a/2-cartpole/5-a3c/save_model/Cartpole_A3C_critic.h5 b/2-cartpole/5-a3c/save_model/Cartpole_A3C_critic.h5 deleted file mode 100644 index 5db01072..00000000 Binary files a/2-cartpole/5-a3c/save_model/Cartpole_A3C_critic.h5 and /dev/null differ diff --git a/2-cartpole/README.md b/2-cartpole/README.md deleted file mode 100644 index 1d8d8701..00000000 --- a/2-cartpole/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# OpenAI gym Cartpole - - -Various reinforcement learning algorithms for Cartpole example. -

- - -
-This is graph of DQN algorithm - -

- -
-This is graph of Double DQN algorithm - -

- -
-This is graph of Policy Gradient algorithm -

- -
-This is graph of Actor Critic algorithm -

\ No newline at end of file diff --git a/2-cartpole/env.py b/2-cartpole/env.py new file mode 100644 index 00000000..3f470589 --- /dev/null +++ b/2-cartpole/env.py @@ -0,0 +1,62 @@ +"""Shared CartPole-v1 setup for the three cartpole algorithms. + +Each algorithm file gets the same --render / --test CLI, the same env +construction, and the same test-mode loop — they differ only in how +they pick an action and how they load their checkpoint. +""" +import argparse +import sys + +import gymnasium as gym +import numpy as np +import pygame + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--render", action="store_true", + help="show the cartpole window during training") + parser.add_argument("--test", action="store_true", + help="load the saved checkpoint and just play (no learning)") + return parser.parse_args() + + +def make_env(args): + return gym.make("CartPole-v1", + render_mode="human" if (args.render or args.test) else None) + + +def quit_if_window_closed(env): + """Exit cleanly when the user clicks the window's X. + + Gymnasium's classic_control renderer pumps pygame's internal event + processing but doesn't act on QUIT, so without this nothing would + happen on close. Safe to call from headless runs too: when no + display is initialized the function returns immediately. + """ + if not pygame.display.get_init(): + return + for event in pygame.event.get(): + if event.type == pygame.QUIT: + env.close() + sys.exit() + + +def run_test_loop(env, get_action): + """Replay episodes forever using the supplied action picker. + + `get_action(state: np.ndarray) -> int`. + """ + while True: + state, _ = env.reset() + state = np.array(state, dtype=np.float32) + done = False + score = 0 + while not done: + quit_if_window_closed(env) + action = get_action(state) + next_state, reward, terminated, truncated, _ = env.step(action) + done = terminated or truncated + state = np.array(next_state, dtype=np.float32) + score += reward + print(f"test score: {score}") diff --git a/3-atari/1-breakout/breakout_a3c.py b/3-atari/1-breakout/breakout_a3c.py deleted file mode 100644 index be339e8e..00000000 --- a/3-atari/1-breakout/breakout_a3c.py +++ /dev/null @@ -1,351 +0,0 @@ -import gym -import time -import random -import threading -import numpy as np -import tensorflow as tf -from skimage.color import rgb2gray -from skimage.transform import resize -from keras.models import Model -from keras.optimizers import RMSprop -from keras.layers import Dense, Flatten, Input -from keras.layers.convolutional import Conv2D -from keras import backend as K - -# global variables for A3C -global episode -episode = 0 -EPISODES = 8000000 -# In case of BreakoutDeterministic-v3, always skip 4 frames -# Deterministic-v4 version use 4 actions -env_name = "BreakoutDeterministic-v4" - -# This is A3C(Asynchronous Advantage Actor Critic) agent(global) for the Cartpole -# In this example, we use A3C algorithm -class A3CAgent: - def __init__(self, action_size): - # environment settings - self.state_size = (84, 84, 4) - self.action_size = action_size - - self.discount_factor = 0.99 - self.no_op_steps = 30 - - # optimizer parameters - self.actor_lr = 2.5e-4 - self.critic_lr = 2.5e-4 - self.threads = 8 - - # create model for actor and critic network - self.actor, self.critic = self.build_model() - - # method for training actor and critic network - self.optimizer = [self.actor_optimizer(), self.critic_optimizer()] - - self.sess = tf.InteractiveSession() - K.set_session(self.sess) - self.sess.run(tf.global_variables_initializer()) - - self.summary_placeholders, self.update_ops, self.summary_op = self.setup_summary() - self.summary_writer = tf.summary.FileWriter('summary/breakout_a3c', self.sess.graph) - - def train(self): - # self.load_model("./save_model/breakout_a3c") - agents = [Agent(self.action_size, self.state_size, [self.actor, self.critic], self.sess, self.optimizer, - self.discount_factor, [self.summary_op, self.summary_placeholders, - self.update_ops, self.summary_writer]) for _ in range(self.threads)] - - for agent in agents: - time.sleep(1) - agent.start() - - while True: - time.sleep(60*10) - self.save_model("./save_model/breakout_a3c") - - # approximate policy and value using Neural Network - # actor -> state is input and probability of each action is output of network - # critic -> state is input and value of state is output of network - # actor and critic network share first hidden layer - def build_model(self): - input = Input(shape=self.state_size) - conv = Conv2D(16, (8, 8), strides=(4, 4), activation='relu')(input) - conv = Conv2D(32, (4, 4), strides=(2, 2), activation='relu')(conv) - conv = Flatten()(conv) - fc = Dense(256, activation='relu')(conv) - policy = Dense(self.action_size, activation='softmax')(fc) - value = Dense(1, activation='linear')(fc) - - actor = Model(inputs=input, outputs=policy) - critic = Model(inputs=input, outputs=value) - - actor._make_predict_function() - critic._make_predict_function() - - actor.summary() - critic.summary() - - return actor, critic - - # make loss function for Policy Gradient - # [log(action probability) * advantages] will be input for the back prop - # we add entropy of action probability to loss - def actor_optimizer(self): - action = K.placeholder(shape=[None, self.action_size]) - advantages = K.placeholder(shape=[None, ]) - - policy = self.actor.output - - good_prob = K.sum(action * policy, axis=1) - eligibility = K.log(good_prob + 1e-10) * advantages - actor_loss = -K.sum(eligibility) - - entropy = K.sum(policy * K.log(policy + 1e-10), axis=1) - entropy = K.sum(entropy) - - loss = actor_loss + 0.01*entropy - optimizer = RMSprop(lr=self.actor_lr, rho=0.99, epsilon=0.01) - updates = optimizer.get_updates(self.actor.trainable_weights, [], loss) - train = K.function([self.actor.input, action, advantages], [loss], updates=updates) - - return train - - # make loss function for Value approximation - def critic_optimizer(self): - discounted_reward = K.placeholder(shape=(None, )) - - value = self.critic.output - - loss = K.mean(K.square(discounted_reward - value)) - - optimizer = RMSprop(lr=self.critic_lr, rho=0.99, epsilon=0.01) - updates = optimizer.get_updates(self.critic.trainable_weights, [], loss) - train = K.function([self.critic.input, discounted_reward], [loss], updates=updates) - return train - - def load_model(self, name): - self.actor.load_weights(name + "_actor.h5") - self.critic.load_weights(name + "_critic.h5") - - def save_model(self, name): - self.actor.save_weights(name + "_actor.h5") - self.critic.save_weights(name + '_critic.h5') - - # make summary operators for tensorboard - def setup_summary(self): - episode_total_reward = tf.Variable(0.) - episode_avg_max_q = tf.Variable(0.) - episode_duration = tf.Variable(0.) - - tf.summary.scalar('Total Reward/Episode', episode_total_reward) - tf.summary.scalar('Average Max Prob/Episode', episode_avg_max_q) - tf.summary.scalar('Duration/Episode', episode_duration) - - summary_vars = [episode_total_reward, episode_avg_max_q, episode_duration] - summary_placeholders = [tf.placeholder(tf.float32) for _ in range(len(summary_vars))] - update_ops = [summary_vars[i].assign(summary_placeholders[i]) for i in range(len(summary_vars))] - summary_op = tf.summary.merge_all() - return summary_placeholders, update_ops, summary_op - -# make agents(local) and start training -class Agent(threading.Thread): - def __init__(self, action_size, state_size, model, sess, optimizer, discount_factor, summary_ops): - threading.Thread.__init__(self) - - self.action_size = action_size - self.state_size = state_size - self.actor, self.critic = model - self.sess = sess - self.optimizer = optimizer - self.discount_factor = discount_factor - self.summary_op, self.summary_placeholders, self.update_ops, self.summary_writer = summary_ops - - self.states, self.actions, self.rewards = [],[],[] - - self.local_actor, self.local_critic = self.build_localmodel() - - self.avg_p_max = 0 - self.avg_loss = 0 - - # t_max -> max batch size for training - self.t_max = 20 - self.t = 0 - - # Thread interactive with environment - def run(self): - # self.load_model('./save_model/breakout_a3c') - global episode - - env = gym.make(env_name) - - step = 0 - - while episode < EPISODES: - done = False - dead = False - # 1 episode = 5 lives - score, start_life = 0, 5 - observe = env.reset() - next_observe = observe - - # this is one of DeepMind's idea. - # just do nothing at the start of episode to avoid sub-optimal - for _ in range(random.randint(1, 30)): - observe = next_observe - next_observe, _, _, _ = env.step(1) - - # At start of episode, there is no preceding frame. So just copy initial states to make history - state = pre_processing(next_observe, observe) - history = np.stack((state, state, state, state), axis=2) - history = np.reshape([history], (1, 84, 84, 4)) - - while not done: - step += 1 - self.t += 1 - observe = next_observe - # get action for the current history and go one step in environment - action, policy = self.get_action(history) - # change action to real_action - if action == 0: real_action = 1 - elif action == 1: real_action = 2 - else: real_action = 3 - - if dead: - action = 0 - real_action = 1 - dead = False - - next_observe, reward, done, info = env.step(real_action) - # pre-process the observation --> history - next_state = pre_processing(next_observe, observe) - next_state = np.reshape([next_state], (1, 84, 84, 1)) - next_history = np.append(next_state, history[:, :, :, :3], axis=3) - - self.avg_p_max += np.amax(self.actor.predict(np.float32(history / 255.))) - - # if the ball is fall, then the agent is dead --> episode is not over - if start_life > info['ale.lives']: - dead = True - start_life = info['ale.lives'] - - score += reward - reward = np.clip(reward, -1., 1.) - - # save the sample to the replay memory - self.memory(history, action, reward) - - # if agent is dead, then reset the history - if dead: - history = np.stack((next_state, next_state, next_state, next_state), axis=2) - history = np.reshape([history], (1, 84, 84, 4)) - else: - history = next_history - - # - if self.t >= self.t_max or done: - self.train_model(done) - self.update_localmodel() - self.t = 0 - - # if done, plot the score over episodes - if done: - episode += 1 - print("episode:", episode, " score:", score, " step:", step) - - stats = [score, self.avg_p_max / float(step), - step] - for i in range(len(stats)): - self.sess.run(self.update_ops[i], feed_dict={ - self.summary_placeholders[i]: float(stats[i]) - }) - summary_str = self.sess.run(self.summary_op) - self.summary_writer.add_summary(summary_str, episode + 1) - self.avg_p_max = 0 - self.avg_loss = 0 - step = 0 - - # In Policy Gradient, Q function is not available. - # Instead agent uses sample returns for evaluating policy - def discount_rewards(self, rewards, done): - discounted_rewards = np.zeros_like(rewards) - running_add = 0 - if not done: - running_add = self.critic.predict(np.float32(self.states[-1] / 255.))[0] - for t in reversed(range(0, len(rewards))): - running_add = running_add * self.discount_factor + rewards[t] - discounted_rewards[t] = running_add - return discounted_rewards - - # update policy network and value network every episode - def train_model(self, done): - discounted_rewards = self.discount_rewards(self.rewards, done) - - states = np.zeros((len(self.states), 84, 84, 4)) - for i in range(len(self.states)): - states[i] = self.states[i] - - states = np.float32(states / 255.) - - values = self.critic.predict(states) - values = np.reshape(values, len(values)) - - advantages = discounted_rewards - values - - self.optimizer[0]([states, self.actions, advantages]) - self.optimizer[1]([states, discounted_rewards]) - self.states, self.actions, self.rewards = [], [], [] - - def build_localmodel(self): - input = Input(shape=self.state_size) - conv = Conv2D(16, (8, 8), strides=(4, 4), activation='relu')(input) - conv = Conv2D(32, (4, 4), strides=(2, 2), activation='relu')(conv) - conv = Flatten()(conv) - fc = Dense(256, activation='relu')(conv) - policy = Dense(self.action_size, activation='softmax')(fc) - value = Dense(1, activation='linear')(fc) - - actor = Model(inputs=input, outputs=policy) - critic = Model(inputs=input, outputs=value) - - actor._make_predict_function() - critic._make_predict_function() - - actor.set_weights(self.actor.get_weights()) - critic.set_weights(self.critic.get_weights()) - - actor.summary() - critic.summary() - - return actor, critic - - def update_localmodel(self): - self.local_actor.set_weights(self.actor.get_weights()) - self.local_critic.set_weights(self.critic.get_weights()) - - def get_action(self, history): - history = np.float32(history / 255.) - policy = self.local_actor.predict(history)[0] - action_index = np.random.choice(self.action_size, 1, p=policy)[0] - return action_index, policy - - # save of each step - # this is used for calculating discounted rewards - def memory(self, history, action, reward): - self.states.append(history) - act = np.zeros(self.action_size) - act[action] = 1 - self.actions.append(act) - self.rewards.append(reward) - - -# 210*160*3(color) --> 84*84(mono) -# float --> integer (to reduce the size of replay memory) -def pre_processing(next_observe, observe): - processed_observe = np.maximum(next_observe, observe) - processed_observe = np.uint8(resize(rgb2gray(processed_observe), (84, 84), mode='constant') * 255) - return processed_observe - - -if __name__ == "__main__": - global_agent = A3CAgent(action_size=3) - global_agent.train() diff --git a/3-atari/1-breakout/breakout_ddqn.py b/3-atari/1-breakout/breakout_ddqn.py deleted file mode 100644 index f9f0a5ed..00000000 --- a/3-atari/1-breakout/breakout_ddqn.py +++ /dev/null @@ -1,274 +0,0 @@ -import gym -import random -import numpy as np -import tensorflow as tf -from collections import deque -from skimage.color import rgb2gray -from skimage.transform import resize -from keras.models import Sequential -from keras.optimizers import RMSprop -from keras.layers import Dense, Flatten -from keras.layers.convolutional import Conv2D -from keras import backend as K - -EPISODES = 50000 - - -class DDQNAgent: - def __init__(self, action_size): - self.render = False - self.load_model = False - # environment settings - self.state_size = (84, 84, 4) - self.action_size = action_size - # parameters about epsilon - self.epsilon = 1. - self.epsilon_start, self.epsilon_end = 1.0, 0.1 - self.exploration_steps = 1000000. - self.epsilon_decay_step = (self.epsilon_start - self.epsilon_end) \ - / self.exploration_steps - # parameters about training - self.batch_size = 32 - self.train_start = 50000 - self.update_target_rate = 10000 - self.discount_factor = 0.99 - self.memory = deque(maxlen=400000) - self.no_op_steps = 30 - # build - self.model = self.build_model() - self.target_model = self.build_model() - self.update_target_model() - - self.optimizer = self.optimizer() - - self.sess = tf.InteractiveSession() - K.set_session(self.sess) - - self.avg_q_max, self.avg_loss = 0, 0 - self.summary_placeholders, self.update_ops, self.summary_op = \ - self.setup_summary() - self.summary_writer = tf.summary.FileWriter( - 'summary/breakout_ddqn', self.sess.graph) - self.sess.run(tf.global_variables_initializer()) - - if self.load_model: - self.model.load_weights("./save_model/breakout_ddqn.h5") - - # if the error is in [-1, 1], then the cost is quadratic to the error - # But outside the interval, the cost is linear to the error - def optimizer(self): - a = K.placeholder(shape=(None, ), dtype='int32') - y = K.placeholder(shape=(None, ), dtype='float32') - - py_x = self.model.output - - a_one_hot = K.one_hot(a, self.action_size) - q_value = K.sum(py_x * a_one_hot, axis=1) - error = K.abs(y - q_value) - - quadratic_part = K.clip(error, 0.0, 1.0) - linear_part = error - quadratic_part - loss = K.mean(0.5 * K.square(quadratic_part) + linear_part) - - optimizer = RMSprop(lr=0.00025, epsilon=0.01) - updates = optimizer.get_updates(self.model.trainable_weights, [], loss) - train = K.function([self.model.input, a, y], [loss], updates=updates) - - return train - - # approximate Q function using Convolution Neural Network - # state is input and Q Value of each action is output of network - def build_model(self): - model = Sequential() - model.add(Conv2D(32, (8, 8), strides=(4, 4), activation='relu', - input_shape=self.state_size)) - model.add(Conv2D(64, (4, 4), strides=(2, 2), activation='relu')) - model.add(Conv2D(64, (3, 3), strides=(1, 1), activation='relu')) - model.add(Flatten()) - model.add(Dense(512, activation='relu')) - model.add(Dense(self.action_size)) - model.summary() - - return model - - # after some time interval update the target model to be same with model - def update_target_model(self): - self.target_model.set_weights(self.model.get_weights()) - - # get action from model using epsilon-greedy policy - def get_action(self, history): - history = np.float32(history / 255.0) - if np.random.rand() <= self.epsilon: - return random.randrange(self.action_size) - else: - q_value = self.model.predict(history) - return np.argmax(q_value[0]) - - # save sample to the replay memory - def replay_memory(self, history, action, reward, next_history, dead): - self.memory.append((history, action, reward, next_history, dead)) - - # pick samples randomly from replay memory (with batch_size) - def train_replay(self): - if len(self.memory) < self.train_start: - return - if self.epsilon > self.epsilon_end: - self.epsilon -= self.epsilon_decay_step - - mini_batch = random.sample(self.memory, self.batch_size) - - history = np.zeros((self.batch_size, self.state_size[0], - self.state_size[1], self.state_size[2])) - next_history = np.zeros((self.batch_size, self.state_size[0], - self.state_size[1], self.state_size[2])) - target = np.zeros((self.batch_size, )) - action, reward, dead = [], [], [] - - for i in range(self.batch_size): - history[i] = np.float32(mini_batch[i][0] / 255.) - next_history[i] = np.float32(mini_batch[i][3] / 255.) - action.append(mini_batch[i][1]) - reward.append(mini_batch[i][2]) - dead.append(mini_batch[i][4]) - - value = self.model.predict(next_history) - target_value = self.target_model.predict(next_history) - - # like Q Learning, get maximum Q value at s' - # But from target model - for i in range(self.batch_size): - if dead[i]: - target[i] = reward[i] - else: - # the key point of Double DQN - # selection of action is from model - # update is from target model - target[i] = reward[i] + self.discount_factor * \ - target_value[i][np.argmax(value[i])] - - loss = self.optimizer([history, action, target]) - self.avg_loss += loss[0] - - # make summary operators for tensorboard - def setup_summary(self): - episode_total_reward = tf.Variable(0.) - episode_avg_max_q = tf.Variable(0.) - episode_duration = tf.Variable(0.) - episode_avg_loss = tf.Variable(0.) - - tf.summary.scalar('Total Reward/Episode', episode_total_reward) - tf.summary.scalar('Average Max Q/Episode', episode_avg_max_q) - tf.summary.scalar('Duration/Episode', episode_duration) - tf.summary.scalar('Average Loss/Episode', episode_avg_loss) - - summary_vars = [episode_total_reward, episode_avg_max_q, - episode_duration, episode_avg_loss] - summary_placeholders = [tf.placeholder(tf.float32) for _ in - range(len(summary_vars))] - update_ops = [summary_vars[i].assign(summary_placeholders[i]) for i in - range(len(summary_vars))] - summary_op = tf.summary.merge_all() - return summary_placeholders, update_ops, summary_op - - -# 210*160*3(color) --> 84*84(mono) -# float --> integer (to reduce the size of replay memory) -def pre_processing(observe): - processed_observe = np.uint8( - resize(rgb2gray(observe), (84, 84), mode='constant') * 255) - return processed_observe - - -if __name__ == "__main__": - # In case of BreakoutDeterministic-v4, always skip 4 frames - # Deterministic-v4 version use 4 actions - env = gym.make('BreakoutDeterministic-v4') - agent = DDQNAgent(action_size=3) - - scores, episodes, global_step = [], [], 0 - - for e in range(EPISODES): - done = False - dead = False - # 1 episode = 5 lives - step, score, start_life = 0, 0, 5 - observe = env.reset() - - # this is one of DeepMind's idea. - # just do nothing at the start of episode to avoid sub-optimal - for _ in range(random.randint(1, agent.no_op_steps)): - observe, _, _, _ = env.step(1) - - # At start of episode, there is no preceding frame. - # So just copy initial states to make history - state = pre_processing(observe) - history = np.stack((state, state, state, state), axis=2) - history = np.reshape([history], (1, 84, 84, 4)) - - while not done: - if agent.render: - env.render() - global_step += 1 - step += 1 - - # get action for the current history and go one step in environment - action = agent.get_action(history) - # change action to real_action - if action == 0: real_action = 1 - elif action == 1: real_action = 2 - else: real_action = 3 - - observe, reward, done, info = env.step(real_action) - # pre-process the observation --> history - next_state = pre_processing(observe) - next_state = np.reshape([next_state], (1, 84, 84, 1)) - next_history = np.append(next_state, history[:, :, :, :3], axis=3) - - agent.avg_q_max += np.amax( - agent.model.predict(np.float32(history / 255.))[0]) - - # if the agent missed ball, agent is dead --> episode is not over - if start_life > info['ale.lives']: - dead = True - start_life = info['ale.lives'] - - reward = np.clip(reward, -1., 1.) - - # save the sample to the replay memory - agent.replay_memory(history, action, reward, next_history, dead) - # every some time interval, train model - agent.train_replay() - # update the target model with model - if global_step % agent.update_target_rate == 0: - agent.update_target_model() - - score += reward - - # if agent is dead, then reset the history - if dead: - dead = False - else: - history = next_history - - # if done, plot the score over episodes - if done: - if global_step > agent.train_start: - stats = [score, agent.avg_q_max / float(step), step, - agent.avg_loss / float(step)] - for i in range(len(stats)): - agent.sess.run(agent.update_ops[i], feed_dict={ - agent.summary_placeholders[i]: float(stats[i]) - }) - summary_str = agent.sess.run(agent.summary_op) - agent.summary_writer.add_summary(summary_str, e + 1) - - print("episode:", e, " score:", score, " memory length:", - len(agent.memory), " epsilon:", agent.epsilon, - " global_step:", global_step, " average_q:", - agent.avg_q_max/float(step), " average loss:", - agent.avg_loss/float(step)) - - agent.avg_q_max, agent.avg_loss = 0, 0 - - if e % 1000 == 0: - agent.model.save_weights("./save_model/breakout_ddqn.h5") diff --git a/3-atari/1-breakout/breakout_dqn.py b/3-atari/1-breakout/breakout_dqn.py deleted file mode 100644 index b6229a04..00000000 --- a/3-atari/1-breakout/breakout_dqn.py +++ /dev/null @@ -1,275 +0,0 @@ -import gym -import random -import numpy as np -import tensorflow as tf -from collections import deque -from skimage.color import rgb2gray -from skimage.transform import resize -from keras.models import Sequential -from keras.optimizers import RMSprop -from keras.layers import Dense, Flatten -from keras.layers.convolutional import Conv2D -from keras import backend as K - -EPISODES = 50000 - - -class DQNAgent: - def __init__(self, action_size): - self.render = False - self.load_model = False - # environment settings - self.state_size = (84, 84, 4) - self.action_size = action_size - # parameters about epsilon - self.epsilon = 1. - self.epsilon_start, self.epsilon_end = 1.0, 0.1 - self.exploration_steps = 1000000. - self.epsilon_decay_step = (self.epsilon_start - self.epsilon_end) \ - / self.exploration_steps - # parameters about training - self.batch_size = 32 - self.train_start = 50000 - self.update_target_rate = 10000 - self.discount_factor = 0.99 - self.memory = deque(maxlen=400000) - self.no_op_steps = 30 - # build model - self.model = self.build_model() - self.target_model = self.build_model() - self.update_target_model() - - self.optimizer = self.optimizer() - - self.sess = tf.InteractiveSession() - K.set_session(self.sess) - - self.avg_q_max, self.avg_loss = 0, 0 - self.summary_placeholders, self.update_ops, self.summary_op = \ - self.setup_summary() - self.summary_writer = tf.summary.FileWriter( - 'summary/breakout_dqn', self.sess.graph) - self.sess.run(tf.global_variables_initializer()) - - if self.load_model: - self.model.load_weights("./save_model/breakout_dqn.h5") - - # if the error is in [-1, 1], then the cost is quadratic to the error - # But outside the interval, the cost is linear to the error - def optimizer(self): - a = K.placeholder(shape=(None,), dtype='int32') - y = K.placeholder(shape=(None,), dtype='float32') - - py_x = self.model.output - - a_one_hot = K.one_hot(a, self.action_size) - q_value = K.sum(py_x * a_one_hot, axis=1) - error = K.abs(y - q_value) - - quadratic_part = K.clip(error, 0.0, 1.0) - linear_part = error - quadratic_part - loss = K.mean(0.5 * K.square(quadratic_part) + linear_part) - - optimizer = RMSprop(lr=0.00025, epsilon=0.01) - updates = optimizer.get_updates(self.model.trainable_weights, [], loss) - train = K.function([self.model.input, a, y], [loss], updates=updates) - - return train - - # approximate Q function using Convolution Neural Network - # state is input and Q Value of each action is output of network - def build_model(self): - model = Sequential() - model.add(Conv2D(32, (8, 8), strides=(4, 4), activation='relu', - input_shape=self.state_size)) - model.add(Conv2D(64, (4, 4), strides=(2, 2), activation='relu')) - model.add(Conv2D(64, (3, 3), strides=(1, 1), activation='relu')) - model.add(Flatten()) - model.add(Dense(512, activation='relu')) - model.add(Dense(self.action_size)) - model.summary() - return model - - # after some time interval update the target model to be same with model - def update_target_model(self): - self.target_model.set_weights(self.model.get_weights()) - - # get action from model using epsilon-greedy policy - def get_action(self, history): - history = np.float32(history / 255.0) - if np.random.rand() <= self.epsilon: - return random.randrange(self.action_size) - else: - q_value = self.model.predict(history) - return np.argmax(q_value[0]) - - # save sample to the replay memory - def replay_memory(self, history, action, reward, next_history, dead): - self.memory.append((history, action, reward, next_history, dead)) - - # pick samples randomly from replay memory (with batch_size) - def train_replay(self): - if len(self.memory) < self.train_start: - return - if self.epsilon > self.epsilon_end: - self.epsilon -= self.epsilon_decay_step - - mini_batch = random.sample(self.memory, self.batch_size) - - history = np.zeros((self.batch_size, self.state_size[0], - self.state_size[1], self.state_size[2])) - next_history = np.zeros((self.batch_size, self.state_size[0], - self.state_size[1], self.state_size[2])) - target = np.zeros((self.batch_size,)) - action, reward, dead = [], [], [] - - for i in range(self.batch_size): - history[i] = np.float32(mini_batch[i][0] / 255.) - next_history[i] = np.float32(mini_batch[i][3] / 255.) - action.append(mini_batch[i][1]) - reward.append(mini_batch[i][2]) - dead.append(mini_batch[i][4]) - - target_value = self.target_model.predict(next_history) - - # like Q Learning, get maximum Q value at s' - # But from target model - for i in range(self.batch_size): - if dead[i]: - target[i] = reward[i] - else: - target[i] = reward[i] + self.discount_factor * \ - np.amax(target_value[i]) - - loss = self.optimizer([history, action, target]) - self.avg_loss += loss[0] - - def save_model(self, name): - self.model.save_weights(name) - - # make summary operators for tensorboard - def setup_summary(self): - episode_total_reward = tf.Variable(0.) - episode_avg_max_q = tf.Variable(0.) - episode_duration = tf.Variable(0.) - episode_avg_loss = tf.Variable(0.) - - tf.summary.scalar('Total Reward/Episode', episode_total_reward) - tf.summary.scalar('Average Max Q/Episode', episode_avg_max_q) - tf.summary.scalar('Duration/Episode', episode_duration) - tf.summary.scalar('Average Loss/Episode', episode_avg_loss) - - summary_vars = [episode_total_reward, episode_avg_max_q, - episode_duration, episode_avg_loss] - summary_placeholders = [tf.placeholder(tf.float32) for _ in - range(len(summary_vars))] - update_ops = [summary_vars[i].assign(summary_placeholders[i]) for i in - range(len(summary_vars))] - summary_op = tf.summary.merge_all() - return summary_placeholders, update_ops, summary_op - - -# 210*160*3(color) --> 84*84(mono) -# float --> integer (to reduce the size of replay memory) -def pre_processing(observe): - processed_observe = np.uint8( - resize(rgb2gray(observe), (84, 84), mode='constant') * 255) - return processed_observe - - -if __name__ == "__main__": - # In case of BreakoutDeterministic-v3, always skip 4 frames - # Deterministic-v4 version use 4 actions - env = gym.make('BreakoutDeterministic-v4') - agent = DQNAgent(action_size=3) - - scores, episodes, global_step = [], [], 0 - - for e in range(EPISODES): - done = False - dead = False - # 1 episode = 5 lives - step, score, start_life = 0, 0, 5 - observe = env.reset() - - # this is one of DeepMind's idea. - # just do nothing at the start of episode to avoid sub-optimal - for _ in range(random.randint(1, agent.no_op_steps)): - observe, _, _, _ = env.step(1) - - # At start of episode, there is no preceding frame - # So just copy initial states to make history - state = pre_processing(observe) - history = np.stack((state, state, state, state), axis=2) - history = np.reshape([history], (1, 84, 84, 4)) - - while not done: - if agent.render: - env.render() - global_step += 1 - step += 1 - - # get action for the current history and go one step in environment - action = agent.get_action(history) - # change action to real_action - if action == 0: - real_action = 1 - elif action == 1: - real_action = 2 - else: - real_action = 3 - - observe, reward, done, info = env.step(real_action) - # pre-process the observation --> history - next_state = pre_processing(observe) - next_state = np.reshape([next_state], (1, 84, 84, 1)) - next_history = np.append(next_state, history[:, :, :, :3], axis=3) - - agent.avg_q_max += np.amax( - agent.model.predict(np.float32(history / 255.))[0]) - - # if the agent missed ball, agent is dead --> episode is not over - if start_life > info['ale.lives']: - dead = True - start_life = info['ale.lives'] - - reward = np.clip(reward, -1., 1.) - - # save the sample to the replay memory - agent.replay_memory(history, action, reward, next_history, dead) - # every some time interval, train model - agent.train_replay() - # update the target model with model - if global_step % agent.update_target_rate == 0: - agent.update_target_model() - - score += reward - - # if agent is dead, then reset the history - if dead: - dead = False - else: - history = next_history - - # if done, plot the score over episodes - if done: - if global_step > agent.train_start: - stats = [score, agent.avg_q_max / float(step), step, - agent.avg_loss / float(step)] - for i in range(len(stats)): - agent.sess.run(agent.update_ops[i], feed_dict={ - agent.summary_placeholders[i]: float(stats[i]) - }) - summary_str = agent.sess.run(agent.summary_op) - agent.summary_writer.add_summary(summary_str, e + 1) - - print("episode:", e, " score:", score, " memory length:", - len(agent.memory), " epsilon:", agent.epsilon, - " global_step:", global_step, " average_q:", - agent.avg_q_max / float(step), " average loss:", - agent.avg_loss / float(step)) - - agent.avg_q_max, agent.avg_loss = 0, 0 - - if e % 1000 == 0: - agent.model.save_weights("./save_model/breakout_dqn.h5") diff --git a/3-atari/1-breakout/breakout_dueling_ddqn.py b/3-atari/1-breakout/breakout_dueling_ddqn.py deleted file mode 100644 index 496b1e05..00000000 --- a/3-atari/1-breakout/breakout_dueling_ddqn.py +++ /dev/null @@ -1,286 +0,0 @@ -import gym -import random -import numpy as np -import tensorflow as tf -from collections import deque -from skimage.color import rgb2gray -from skimage.transform import resize -from keras.models import Model -from keras.optimizers import RMSprop -from keras.layers import Input, Dense, Flatten, Lambda, merge -from keras.layers.convolutional import Conv2D -from keras import backend as K - -EPISODES = 50000 - - -class DuelingDDQNAgent: - def __init__(self, action_size): - self.render = False - self.load_model = False - # environment settings - self.state_size = (84, 84, 4) - self.action_size = action_size - # parameters about epsilon - self.epsilon = 1. - self.epsilon_start, self.epsilon_end = 1.0, 0.1 - self.exploration_steps = 1000000. - self.epsilon_decay_step = (self.epsilon_start - self.epsilon_end) \ - / self.exploration_steps - # parameters about training - self.batch_size = 32 - self.train_start = 50000 - self.update_target_rate = 10000 - self.discount_factor = 0.99 - self.memory = deque(maxlen=400000) - self.no_op_steps = 30 - # build - self.model = self.build_model() - self.target_model = self.build_model() - self.update_target_model() - - self.optimizer = self.optimizer() - - self.sess = tf.InteractiveSession() - K.set_session(self.sess) - - self.avg_q_max, self.avg_loss = 0, 0 - self.summary_placeholders, self.update_ops, self.summary_op = \ - self.setup_summary() - self.summary_writer = tf.summary.FileWriter( - 'summary/breakout_dueling_ddqn', self.sess.graph) - self.sess.run(tf.global_variables_initializer()) - - if self.load_model: - self.model.load_weights("./save_model/breakout_dueling_ddqb.h5") - - # if the error is in [-1, 1], then the cost is quadratic to the error - # But outside the interval, the cost is linear to the error - def optimizer(self): - a = K.placeholder(shape=(None, ), dtype='int32') - y = K.placeholder(shape=(None, ), dtype='float32') - - py_x = self.model.output - - a_one_hot = K.one_hot(a, self.action_size) - q_value = K.sum(py_x * a_one_hot, axis=1) - error = K.abs(y - q_value) - - quadratic_part = K.clip(error, 0.0, 1.0) - linear_part = error - quadratic_part - loss = K.mean(0.5 * K.square(quadratic_part) + linear_part) - - optimizer = RMSprop(lr=0.00025, epsilon=0.01) - updates = optimizer.get_updates(self.model.trainable_weights, [], loss) - train = K.function([self.model.input, a, y], [loss], updates=updates) - - return train - - # approximate Q function using Convolution Neural Network - # state is input and Q Value of each action is output of network - # dueling network's Q Value is sum of advantages and state value - def build_model(self): - input = Input(shape=self.state_size) - shared = Conv2D(32, (8, 8), strides=(4, 4), activation='relu')(input) - shared = Conv2D(64, (4, 4), strides=(2, 2), activation='relu')(shared) - shared = Conv2D(64, (3, 3), strides=(1, 1), activation='relu')(shared) - flatten = Flatten()(shared) - - # network separate state value and advantages - advantage_fc = Dense(512, activation='relu')(flatten) - advantage = Dense(self.action_size)(advantage_fc) - advantage = Lambda(lambda a: a[:, :] - K.mean(a[:, :], keepdims=True), - output_shape=(self.action_size,))(advantage) - - value_fc = Dense(512, activation='relu')(flatten) - value = Dense(1)(value_fc) - value = Lambda(lambda s: K.expand_dims(s[:, 0], -1), - output_shape=(self.action_size,))(value) - - # network merged and make Q Value - q_value = merge([value, advantage], mode='sum') - model = Model(inputs=input, outputs=q_value) - model.summary() - - return model - - # after some time interval update the target model to be same with model - def update_target_model(self): - self.target_model.set_weights(self.model.get_weights()) - - # get action from model using epsilon-greedy policy - def get_action(self, history): - history = np.float32(history / 255.0) - if np.random.rand() <= self.epsilon: - return random.randrange(self.action_size) - else: - q_value = self.model.predict(history) - return np.argmax(q_value[0]) - - # save sample to the replay memory - def replay_memory(self, history, action, reward, next_history, dead): - self.memory.append((history, action, reward, next_history, dead)) - - # pick samples randomly from replay memory (with batch_size) - def train_replay(self): - if len(self.memory) < self.train_start: - return - if self.epsilon > self.epsilon_end: - self.epsilon -= self.epsilon_decay_step - - mini_batch = random.sample(self.memory, self.batch_size) - - history = np.zeros((self.batch_size, self.state_size[0], - self.state_size[1], self.state_size[2])) - next_history = np.zeros((self.batch_size, self.state_size[0], - self.state_size[1], self.state_size[2])) - target = np.zeros((self.batch_size, )) - action, reward, dead = [], [], [] - - for i in range(self.batch_size): - history[i] = np.float32(mini_batch[i][0] / 255.) - next_history[i] = np.float32(mini_batch[i][3] / 255.) - action.append(mini_batch[i][1]) - reward.append(mini_batch[i][2]) - dead.append(mini_batch[i][4]) - - value = self.model.predict(history) - target_value = self.target_model.predict(next_history) - - # like Q Learning, get maximum Q value at s' - # But from target model - for i in range(self.batch_size): - if dead[i]: - target[i] = reward[i] - else: - # the key point of Double DQN - # selection of action is from model - # update is from target model - target[i] = reward[i] + self.discount_factor * \ - target_value[i][np.argmax(value[i])] - - loss = self.optimizer([history, action, target]) - self.avg_loss += loss[0] - - def setup_summary(self): - episode_total_reward = tf.Variable(0.) - episode_avg_max_q = tf.Variable(0.) - episode_duration = tf.Variable(0.) - episode_avg_loss = tf.Variable(0.) - - tf.summary.scalar('Total Reward/Episode', episode_total_reward) - tf.summary.scalar('Average Max Q/Episode', episode_avg_max_q) - tf.summary.scalar('Duration/Episode', episode_duration) - tf.summary.scalar('Average Loss/Episode', episode_avg_loss) - - summary_vars = [episode_total_reward, episode_avg_max_q, - episode_duration, episode_avg_loss] - summary_placeholders = [tf.placeholder(tf.float32) for _ in - range(len(summary_vars))] - update_ops = [summary_vars[i].assign(summary_placeholders[i]) for i in - range(len(summary_vars))] - summary_op = tf.summary.merge_all() - return summary_placeholders, update_ops, summary_op - - -# 210*160*3(color) --> 84*84(mono) -# float --> integer (to reduce the size of replay memory) -def pre_processing(observe): - processed_observe = np.uint8( - resize(rgb2gray(observe), (84, 84), mode='constant') * 255) - return processed_observe - - -if __name__ == "__main__": - # In case of BreakoutDeterministic-v3, always skip 4 frames - # Deterministic-v4 version use 4 actions - env = gym.make('BreakoutDeterministic-v4') - agent = DuelingDDQNAgent(action_size=3) - - scores, episodes, global_step = [], [], 0 - - for e in range(EPISODES): - done = False - dead = False - # 1 episode = 5 lives - step, score, start_life = 0, 0, 5 - observe = env.reset() - - # this is one of DeepMind's idea. - # just do nothing at the start of episode to avoid sub-optimal - for _ in range(random.randint(1, agent.no_op_steps)): - observe, _, _, _ = env.step(1) - - # At start of episode, there is no preceding frame. - # So just copy initial states to make history - state = pre_processing(observe) - history = np.stack((state, state, state, state), axis=2) - history = np.reshape([history], (1, 84, 84, 4)) - - while not done: - if agent.render: - env.render() - global_step += 1 - step += 1 - - # get action for the current history and go one step in environment - action = agent.get_action(history) - # change action to real_action - if action == 0: real_action = 1 - elif action == 1: real_action = 2 - else: real_action = 3 - - observe, reward, done, info = env.step(real_action) - # pre-process the observation --> history - next_state = pre_processing(observe) - next_state = np.reshape([next_state], (1, 84, 84, 1)) - next_history = np.append(next_state, history[:, :, :, :3], axis=3) - - agent.avg_q_max += np.amax( - agent.model.predict(np.float32(history / 255.))[0]) - - # if the agent missed ball, agent is dead --> episode is not over - if start_life > info['ale.lives']: - dead = True - start_life = info['ale.lives'] - - reward = np.clip(reward, -1., 1.) - - # save the sample to the replay memory - agent.replay_memory(history, action, reward, next_history, dead) - # every some time interval, train model - agent.train_replay() - # update the target model with model - if global_step % agent.update_target_rate == 0: - agent.update_target_model() - - score += reward - - # if agent is dead, then reset the history - if dead: - dead = False - else: - history = next_history - - # if done, plot the score over episodes - if done: - if global_step > agent.train_start: - stats = [score, agent.avg_q_max / float(step), step, - agent.avg_loss / float(step)] - for i in range(len(stats)): - agent.sess.run(agent.update_ops[i], feed_dict={ - agent.summary_placeholders[i]: float(stats[i]) - }) - summary_str = agent.sess.run(agent.summary_op) - agent.summary_writer.add_summary(summary_str, e + 1) - - print("episode:", e, " score:", score, " memory length:", - len(agent.memory), " epsilon:", agent.epsilon, - " global_step:", global_step, " average_q:", - agent.avg_q_max/float(step), " average loss:", - agent.avg_loss/float(step)) - - agent.avg_q_max, agent.avg_loss = 0, 0 - - if e % 1000 == 0: - agent.model.save_weights("./save_model/breakout_dueling_ddqn.h5") diff --git a/3-atari/1-breakout/play_a3c_model.py b/3-atari/1-breakout/play_a3c_model.py deleted file mode 100644 index c6a32c83..00000000 --- a/3-atari/1-breakout/play_a3c_model.py +++ /dev/null @@ -1,125 +0,0 @@ -import gym -import random -import numpy as np -from skimage.color import rgb2gray -from skimage.transform import resize -from keras.models import Model -from keras.layers import Dense, Flatten, Input -from keras.layers.convolutional import Conv2D - -global episode -episode = 0 -EPISODES = 8000000 -env_name = "BreakoutDeterministic-v4" - -class TestAgent: - def __init__(self, action_size): - self.state_size = (84, 84, 4) - self.action_size = action_size - - self.discount_factor = 0.99 - self.no_op_steps = 30 - - self.actor, self.critic = self.build_model() - - def build_model(self): - input = Input(shape=self.state_size) - conv = Conv2D(16, (8, 8), strides=(4, 4), activation='relu')(input) - conv = Conv2D(32, (4, 4), strides=(2, 2), activation='relu')(conv) - conv = Flatten()(conv) - fc = Dense(256, activation='relu')(conv) - policy = Dense(self.action_size, activation='softmax')(fc) - value = Dense(1, activation='linear')(fc) - - actor = Model(inputs=input, outputs=policy) - critic = Model(inputs=input, outputs=value) - - actor.summary() - critic.summary() - - return actor, critic - - def get_action(self, history): - history = np.float32(history / 255.) - policy = self.actor.predict(history)[0] - - action_index = np.argmax(policy) - return action_index - - def load_model(self, name): - self.actor.load_weights(name) - -def pre_processing(next_observe, observe): - processed_observe = np.maximum(next_observe, observe) - processed_observe = np.uint8( - resize(rgb2gray(processed_observe), (84, 84), mode='constant') * 255) - return processed_observe - - -if __name__ == "__main__": - env = gym.make(env_name) - agent = TestAgent(action_size=3) - agent.load_model("save_model/breakout_a3c_5_actor.h5") - - step = 0 - - while episode < EPISODES: - done = False - dead = False - - score, start_life = 0, 5 - observe = env.reset() - next_observe = observe - - for _ in range(random.randint(1, 20)): - observe = next_observe - next_observe, _, _, _ = env.step(1) - - state = pre_processing(next_observe, observe) - history = np.stack((state, state, state, state), axis=2) - history = np.reshape([history], (1, 84, 84, 4)) - - while not done: - env.render() - step += 1 - observe = next_observe - - action = agent.get_action(history) - - if action == 1: - fake_action = 2 - elif action == 2: - fake_action = 3 - else: - fake_action = 1 - - if dead: - fake_action = 1 - dead = False - - next_observe, reward, done, info = env.step(fake_action) - - next_state = pre_processing(next_observe, observe) - next_state = np.reshape([next_state], (1, 84, 84, 1)) - next_history = np.append(next_state, history[:, :, :, :3], axis=3) - - if start_life > info['ale.lives']: - dead = True - reward = -1 - start_life = info['ale.lives'] - - score += reward - - # if agent is dead, then reset the history - if dead: - history = np.stack( - (next_state, next_state, next_state, next_state), axis=2) - history = np.reshape([history], (1, 84, 84, 4)) - else: - history = next_history - - # if done, plot the score over episodes - if done: - episode += 1 - print("episode:", episode, " score:", score, " step:", step) - step = 0 \ No newline at end of file diff --git a/3-atari/1-breakout/play_dqn_model.py b/3-atari/1-breakout/play_dqn_model.py deleted file mode 100644 index 45662c78..00000000 --- a/3-atari/1-breakout/play_dqn_model.py +++ /dev/null @@ -1,110 +0,0 @@ -import gym -import random -import numpy as np -import tensorflow as tf -from skimage.color import rgb2gray -from skimage.transform import resize -from keras.models import Sequential -from keras.layers import Dense, Flatten -from keras.layers.convolutional import Conv2D -from keras import backend as K - -EPISODES = 50000 - - -class TestAgent: - def __init__(self, action_size): - self.state_size = (84, 84, 4) - self.action_size = action_size - self.no_op_steps = 20 - - self.model = self.build_model() - - self.sess = tf.InteractiveSession() - K.set_session(self.sess) - - self.avg_q_max, self.avg_loss = 0, 0 - self.sess.run(tf.global_variables_initializer()) - - def build_model(self): - model = Sequential() - model.add(Conv2D(32, (8, 8), strides=(4, 4), activation='relu', - input_shape=self.state_size)) - model.add(Conv2D(64, (4, 4), strides=(2, 2), activation='relu')) - model.add(Conv2D(64, (3, 3), strides=(1, 1), activation='relu')) - model.add(Flatten()) - model.add(Dense(512, activation='relu')) - model.add(Dense(self.action_size)) - model.summary() - - return model - - def get_action(self, history): - if np.random.random() < 0.01: - return random.randrange(3) - history = np.float32(history / 255.0) - q_value = self.model.predict(history) - return np.argmax(q_value[0]) - - def load_model(self, filename): - self.model.load_weights(filename) - -def pre_processing(observe): - processed_observe = np.uint8( - resize(rgb2gray(observe), (84, 84), mode='constant') * 255) - return processed_observe - - -if __name__ == "__main__": - env = gym.make('BreakoutDeterministic-v4') - agent = TestAgent(action_size=3) - agent.load_model("./save_model/breakout_dqn_5.h5") - - for e in range(EPISODES): - done = False - dead = False - - step, score, start_life = 0, 0, 5 - observe = env.reset() - - for _ in range(random.randint(1, agent.no_op_steps)): - observe, _, _, _ = env.step(1) - - state = pre_processing(observe) - history = np.stack((state, state, state, state), axis=2) - history = np.reshape([history], (1, 84, 84, 4)) - - while not done: - env.render() - step += 1 - - action = agent.get_action(history) - - if action == 0: - real_action = 1 - elif action == 1: - real_action = 2 - else: - real_action = 3 - - if dead: - real_action = 1 - dead = False - - observe, reward, done, info = env.step(real_action) - - next_state = pre_processing(observe) - next_state = np.reshape([next_state], (1, 84, 84, 1)) - next_history = np.append(next_state, history[:, :, :, :3], axis=3) - - if start_life > info['ale.lives']: - dead = True - start_life = info['ale.lives'] - - score += reward - - history = next_history - - if done: - print("episode:", e, " score:", score) - diff --git a/3-atari/1-breakout/save_model/breakout_a3c_1_actor.h5 b/3-atari/1-breakout/save_model/breakout_a3c_1_actor.h5 deleted file mode 100644 index 37a6a1ac..00000000 Binary files a/3-atari/1-breakout/save_model/breakout_a3c_1_actor.h5 and /dev/null differ diff --git a/3-atari/1-breakout/save_model/breakout_a3c_1_critic.h5 b/3-atari/1-breakout/save_model/breakout_a3c_1_critic.h5 deleted file mode 100644 index 3d3394ae..00000000 Binary files a/3-atari/1-breakout/save_model/breakout_a3c_1_critic.h5 and /dev/null differ diff --git a/3-atari/1-breakout/save_model/breakout_a3c_2_actor.h5 b/3-atari/1-breakout/save_model/breakout_a3c_2_actor.h5 deleted file mode 100644 index 21207c0f..00000000 Binary files a/3-atari/1-breakout/save_model/breakout_a3c_2_actor.h5 and /dev/null differ diff --git a/3-atari/1-breakout/save_model/breakout_a3c_2_critic.h5 b/3-atari/1-breakout/save_model/breakout_a3c_2_critic.h5 deleted file mode 100644 index a26f7d8a..00000000 Binary files a/3-atari/1-breakout/save_model/breakout_a3c_2_critic.h5 and /dev/null differ diff --git a/3-atari/1-breakout/save_model/breakout_a3c_3_actor.h5 b/3-atari/1-breakout/save_model/breakout_a3c_3_actor.h5 deleted file mode 100644 index a27e766e..00000000 Binary files a/3-atari/1-breakout/save_model/breakout_a3c_3_actor.h5 and /dev/null differ diff --git a/3-atari/1-breakout/save_model/breakout_a3c_3_critic.h5 b/3-atari/1-breakout/save_model/breakout_a3c_3_critic.h5 deleted file mode 100644 index 62236fc7..00000000 Binary files a/3-atari/1-breakout/save_model/breakout_a3c_3_critic.h5 and /dev/null differ diff --git a/3-atari/1-breakout/save_model/breakout_a3c_4_actor.h5 b/3-atari/1-breakout/save_model/breakout_a3c_4_actor.h5 deleted file mode 100644 index 4fc3b773..00000000 Binary files a/3-atari/1-breakout/save_model/breakout_a3c_4_actor.h5 and /dev/null differ diff --git a/3-atari/1-breakout/save_model/breakout_a3c_4_critic.h5 b/3-atari/1-breakout/save_model/breakout_a3c_4_critic.h5 deleted file mode 100644 index f65494da..00000000 Binary files a/3-atari/1-breakout/save_model/breakout_a3c_4_critic.h5 and /dev/null differ diff --git a/3-atari/1-breakout/save_model/breakout_a3c_5_actor.h5 b/3-atari/1-breakout/save_model/breakout_a3c_5_actor.h5 deleted file mode 100644 index db855b24..00000000 Binary files a/3-atari/1-breakout/save_model/breakout_a3c_5_actor.h5 and /dev/null differ diff --git a/3-atari/1-breakout/save_model/breakout_a3c_5_critic.h5 b/3-atari/1-breakout/save_model/breakout_a3c_5_critic.h5 deleted file mode 100644 index 3636d02d..00000000 Binary files a/3-atari/1-breakout/save_model/breakout_a3c_5_critic.h5 and /dev/null differ diff --git a/3-atari/1-breakout/save_model/breakout_dqn.h5 b/3-atari/1-breakout/save_model/breakout_dqn.h5 deleted file mode 100644 index fec05377..00000000 Binary files a/3-atari/1-breakout/save_model/breakout_dqn.h5 and /dev/null differ diff --git a/3-atari/1-breakout/save_model/breakout_dqn_1.h5 b/3-atari/1-breakout/save_model/breakout_dqn_1.h5 deleted file mode 100644 index bb219b8a..00000000 Binary files a/3-atari/1-breakout/save_model/breakout_dqn_1.h5 and /dev/null differ diff --git a/3-atari/1-breakout/save_model/breakout_dqn_2.h5 b/3-atari/1-breakout/save_model/breakout_dqn_2.h5 deleted file mode 100644 index f316b4bc..00000000 Binary files a/3-atari/1-breakout/save_model/breakout_dqn_2.h5 and /dev/null differ diff --git a/3-atari/1-breakout/save_model/breakout_dqn_3.h5 b/3-atari/1-breakout/save_model/breakout_dqn_3.h5 deleted file mode 100644 index 3e9ab26d..00000000 Binary files a/3-atari/1-breakout/save_model/breakout_dqn_3.h5 and /dev/null differ diff --git a/3-atari/1-breakout/save_model/breakout_dqn_4.h5 b/3-atari/1-breakout/save_model/breakout_dqn_4.h5 deleted file mode 100644 index 2c952d42..00000000 Binary files a/3-atari/1-breakout/save_model/breakout_dqn_4.h5 and /dev/null differ diff --git a/3-atari/1-breakout/save_model/breakout_dqn_5.h5 b/3-atari/1-breakout/save_model/breakout_dqn_5.h5 deleted file mode 100644 index eae4c99b..00000000 Binary files a/3-atari/1-breakout/save_model/breakout_dqn_5.h5 and /dev/null differ diff --git a/3-atari/1-breakout/summary/breakout_a3c/events.out.tfevents.1497264638 b/3-atari/1-breakout/summary/breakout_a3c/events.out.tfevents.1497264638 deleted file mode 100644 index 1eb4343a..00000000 Binary files a/3-atari/1-breakout/summary/breakout_a3c/events.out.tfevents.1497264638 and /dev/null differ diff --git a/3-atari/1-breakout/summary/breakout_dqn/events.out.tfevents.1496968668.young-System-Product-Name b/3-atari/1-breakout/summary/breakout_dqn/events.out.tfevents.1496968668.young-System-Product-Name deleted file mode 100644 index 2e394adf..00000000 Binary files a/3-atari/1-breakout/summary/breakout_dqn/events.out.tfevents.1496968668.young-System-Product-Name and /dev/null differ diff --git a/3-atari/1-dqn.py b/3-atari/1-dqn.py new file mode 100644 index 00000000..9db4fce7 --- /dev/null +++ b/3-atari/1-dqn.py @@ -0,0 +1,230 @@ +"""DQN agent for Atari (Breakout / Pong). + +Mnih et al., 2015: "Human-level control through deep reinforcement +learning" (Nature). Same algorithm as the cartpole DQN but with the +Nature CNN backbone, a much bigger replay buffer, reward clipping, and +a slower target-network refresh interval. + +Real Atari runs take tens of millions of frames to converge; the +defaults below are tuned to be *runnable* on a laptop rather than to +hit DeepMind-paper scores. Bump TOTAL_FRAMES and BUFFER_CAPACITY for +serious training. +""" +import random +from collections import deque + +import numpy as np +import torch +import torch.nn as nn +import torch.optim as optim + +from env import make_env, parse_args, pick_device, quit_if_window_closed, run_test_loop + + +SAVE_PATH = "atari_dqn.pt" +TOTAL_FRAMES = 10_000_000 # Nature uses 50M agent steps; 10M is laptop-friendly +BUFFER_CAPACITY = 500_000 # ~3.5GB RAM (uint8, single frames stacked at sample time); sized for 8GB Macs +BATCH_SIZE = 32 +GAMMA = 0.99 +LR = 1e-4 +LEARN_START = 80_000 # frames of pure exploration before training begins +TRAIN_EVERY = 4 +TARGET_UPDATE_EVERY = 250 # in training steps, not env steps (~1k env frames) +EPSILON_START = 1.0 +EPSILON_END = 0.01 +EPSILON_DECAY_FRAMES = 1_000_000 # linear decay from start to end over this many frames + + +# Standard Nature CNN. +class QNetwork(nn.Module): + def __init__(self, n_actions): + super().__init__() + self.conv = nn.Sequential( + nn.Conv2d(4, 32, kernel_size=8, stride=4), nn.ReLU(), + nn.Conv2d(32, 64, kernel_size=4, stride=2), nn.ReLU(), + nn.Conv2d(64, 64, kernel_size=3, stride=1), nn.ReLU(), + nn.Flatten(), + ) + self.fc = nn.Sequential( + nn.Linear(64 * 7 * 7, 512), nn.ReLU(), + nn.Linear(512, n_actions), + ) + + def forward(self, x): + # Inputs are uint8 in [0, 255]; normalize on the GPU to save bus bandwidth. + return self.fc(self.conv(x.float() / 255.0)) + + +class ReplayBuffer: + """Single-frame uint8 buffer — stacks of 4 are reconstructed at sample time, + cutting RAM ~4x vs. storing the full stack per slot.""" + + def __init__(self, capacity, frame_shape=(84, 84), stack=4): + self.capacity = capacity + self.stack = stack + self.frames = np.zeros((capacity, *frame_shape), dtype=np.uint8) + self.actions = np.zeros(capacity, dtype=np.int64) + self.rewards = np.zeros(capacity, dtype=np.float32) + self.dones = np.zeros(capacity, dtype=np.float32) + self.idx = 0 + self.size = 0 + + def push(self, frame, action, reward, done): + self.frames[self.idx] = frame + self.actions[self.idx] = action + self.rewards[self.idx] = reward + self.dones[self.idx] = float(done) + self.idx = (self.idx + 1) % self.capacity + self.size = min(self.size + 1, self.capacity) + + def _stack(self, idx): + # Gather frames[idx-stack+1 .. idx]; newest at last channel. + offsets = np.arange(self.stack) + gather = (idx[:, None] - (self.stack - 1) + offsets[None, :]) % self.capacity + out = self.frames[gather] + # Zero out frames sitting before an episode boundary inside the stack. + # dones at the (stack-1) older positions mark where a prior episode ended. + older = self.dones[gather[:, :-1]].astype(bool) + # Once we cross any done walking newest→oldest, everything older is invalid. + invalid = np.cumsum(older[:, ::-1], axis=1)[:, ::-1] > 0 + mask = np.concatenate([~invalid, np.ones((idx.shape[0], 1), dtype=bool)], axis=1) + return out * mask[:, :, None, None] + + def sample(self, batch_size, device): + # Reject indices whose stack would straddle the write head (stale frames). + while True: + if self.size < self.capacity: + if self.size < self.stack + 2: + raise RuntimeError("buffer too small to sample yet") + idx = np.random.randint(self.stack - 1, self.size - 1, size=batch_size) + break + idx = np.random.randint(0, self.capacity, size=batch_size) + dist = (self.idx - 1 - idx) % self.capacity + if np.all(dist >= self.stack): + break + states = self._stack(idx) + next_states = self._stack((idx + 1) % self.capacity) + return ( + torch.as_tensor(states, device=device), + torch.as_tensor(self.actions[idx], device=device), + torch.as_tensor(self.rewards[idx], device=device), + torch.as_tensor(next_states, device=device), + torch.as_tensor(self.dones[idx], device=device), + ) + + +def epsilon(frame): + """Linear schedule from EPSILON_START to EPSILON_END over EPSILON_DECAY_FRAMES.""" + frac = min(frame / EPSILON_DECAY_FRAMES, 1.0) + return EPSILON_START + frac * (EPSILON_END - EPSILON_START) + + +if __name__ == "__main__": + args = parse_args() + device = pick_device(args.device) + env = make_env(args) + n_actions = env.action_space.n + + online = QNetwork(n_actions).to(device) + target = QNetwork(n_actions).to(device) + target.load_state_dict(online.state_dict()) + optimizer = optim.Adam(online.parameters(), lr=LR) + loss_fn = nn.SmoothL1Loss() # Huber loss — standard for DQN + + def greedy_action(obs): + """Used by --test and during exploitation steps.""" + with torch.no_grad(): + t = torch.as_tensor(np.asarray(obs), device=device).unsqueeze(0) + return int(online(t).argmax(dim=1).item()) + + if args.test: + online.load_state_dict(torch.load(SAVE_PATH, map_location=device)) + run_test_loop(env, greedy_action) + + if args.wandb: + import wandb + wandb.init(project="rl-atari-dqn", config={ + "env": args.env, "total_frames": TOTAL_FRAMES, + "buffer_capacity": BUFFER_CAPACITY, "batch_size": BATCH_SIZE, + "gamma": GAMMA, "lr": LR, "learn_start": LEARN_START, + "train_every": TRAIN_EVERY, "target_update_every": TARGET_UPDATE_EVERY, + "epsilon_start": EPSILON_START, "epsilon_end": EPSILON_END, + "epsilon_decay_frames": EPSILON_DECAY_FRAMES, + }) + + print(f"device: {device}, env: {args.env}, actions: {n_actions}") + + buffer = ReplayBuffer(BUFFER_CAPACITY) + obs, _ = env.reset() + ep_return = 0.0 # accumulates within one life (LifeLossTerminalEnv ends an "episode" per life) + game_return = 0.0 # accumulates across all 5 lives until real game-over + recent_returns = deque(maxlen=20) + recent_game_returns = deque(maxlen=20) + train_step = 0 + last_loss = 0.0 + + for frame in range(1, TOTAL_FRAMES + 1): + quit_if_window_closed(env) + + # Epsilon-greedy action. + if random.random() < epsilon(frame): + action = env.action_space.sample() + else: + action = greedy_action(obs) + + next_obs, reward, terminated, truncated, info = env.step(action) + done = terminated or truncated + # Reward clipping (DeepMind standard) — keeps Q-values from blowing up + # when one game has rewards in tens and another in hundreds. + clipped = np.sign(reward) + # FrameStack gives (4, 84, 84); store just the newest frame and stack at sample time. + buffer.push(np.asarray(obs)[-1], action, clipped, done) + + ep_return += reward + game_return += reward + obs = next_obs + if done: + recent_returns.append(ep_return) + ep_return = 0.0 + if info.get("game_over", True): + recent_game_returns.append(game_return) + game_return = 0.0 + obs, _ = env.reset() + + # Training. + if frame > LEARN_START and frame % TRAIN_EVERY == 0: + states, actions, rewards, next_states, dones = buffer.sample(BATCH_SIZE, device) + q_pred = online(states).gather(1, actions.unsqueeze(1)).squeeze(1) + with torch.no_grad(): + q_next = target(next_states).max(dim=1).values + y = rewards + (1.0 - dones) * GAMMA * q_next + loss = loss_fn(q_pred, y) + optimizer.zero_grad() + loss.backward() + # Gradient clipping — DeepMind uses global norm 10. + nn.utils.clip_grad_norm_(online.parameters(), 10.0) + optimizer.step() + last_loss = loss.item() + + train_step += 1 + if train_step % TARGET_UPDATE_EVERY == 0: + target.load_state_dict(online.state_dict()) + + # Logging. + if frame % 10_000 == 0: + mean = float(np.mean(recent_returns)) if recent_returns else 0.0 + game_mean = float(np.mean(recent_game_returns)) if recent_game_returns else 0.0 + print(f"frame: {frame:>8} eps: {epsilon(frame):.3f} " + f"per_life: {mean:.1f} per_game: {game_mean:.1f} buffer: {buffer.size}") + if args.wandb: + wandb.log({ + "global_step": frame, + "recent_mean_return": mean, + "recent_mean_game_return": game_mean, + "epsilon": epsilon(frame), + "loss": last_loss, + "buffer_size": buffer.size, + }, step=frame) + + torch.save(online.state_dict(), SAVE_PATH) + print(f"Saved trained model to {SAVE_PATH}") diff --git a/3-atari/2-pong/README.md b/3-atari/2-pong/README.md deleted file mode 100644 index ed36684b..00000000 --- a/3-atari/2-pong/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# Policy Gradient - -Minimal implementation of Stochastic Policy Gradient Algorithm in Keras - -## Pong Agent - -![pg](assetsg.gif) - - -This PG agent seems to get more frequent wins after about 8000 episodes. Below is the score graph. - - -![score](assetscore.png) diff --git a/3-atari/2-pong/assets/pg.gif b/3-atari/2-pong/assets/pg.gif deleted file mode 100644 index 9a1ac01d..00000000 Binary files a/3-atari/2-pong/assets/pg.gif and /dev/null differ diff --git a/3-atari/2-pong/assets/score.png b/3-atari/2-pong/assets/score.png deleted file mode 100644 index dfb62f87..00000000 Binary files a/3-atari/2-pong/assets/score.png and /dev/null differ diff --git a/3-atari/2-pong/pong_a3c.py b/3-atari/2-pong/pong_a3c.py deleted file mode 100644 index e69de29b..00000000 diff --git a/3-atari/2-pong/pong_reinforce.py b/3-atari/2-pong/pong_reinforce.py deleted file mode 100644 index ce346a78..00000000 --- a/3-atari/2-pong/pong_reinforce.py +++ /dev/null @@ -1,117 +0,0 @@ -import gym -import numpy as np -from keras.models import Sequential -from keras.layers import Dense, Reshape, Flatten -from keras.optimizers import Adam -from keras.layers.convolutional import Convolution2D - - -class PGAgent: - def __init__(self, state_size, action_size): - self.state_size = state_size - self.action_size = action_size - self.gamma = 0.99 - self.learning_rate = 0.001 - self.states = [] - self.gradients = [] - self.rewards = [] - self.probs = [] - self.model = self._build_model() - self.model.summary() - - def _build_model(self): - model = Sequential() - model.add(Reshape((1, 80, 80), input_shape=(self.state_size,))) - model.add(Convolution2D(32, 6, 6, subsample=(3, 3), border_mode='same', - activation='relu', init='he_uniform')) - model.add(Flatten()) - model.add(Dense(64, activation='relu', init='he_uniform')) - model.add(Dense(32, activation='relu', init='he_uniform')) - model.add(Dense(self.action_size, activation='softmax')) - opt = Adam(lr=self.learning_rate) - # See note regarding crossentropy in cartpole_reinforce.py - model.compile(loss='categorical_crossentropy', optimizer=opt) - return model - - def remember(self, state, action, prob, reward): - y = np.zeros([self.action_size]) - y[action] = 1 - self.gradients.append(np.array(y).astype('float32') - prob) - self.states.append(state) - self.rewards.append(reward) - - def act(self, state): - state = state.reshape([1, state.shape[0]]) - aprob = self.model.predict(state, batch_size=1).flatten() - self.probs.append(aprob) - prob = aprob / np.sum(aprob) - action = np.random.choice(self.action_size, 1, p=prob)[0] - return action, prob - - def discount_rewards(self, rewards): - discounted_rewards = np.zeros_like(rewards) - running_add = 0 - for t in reversed(range(0, rewards.size)): - if rewards[t] != 0: - running_add = 0 - running_add = running_add * self.gamma + rewards[t] - discounted_rewards[t] = running_add - return discounted_rewards - - def train(self): - gradients = np.vstack(self.gradients) - rewards = np.vstack(self.rewards) - rewards = self.discount_rewards(rewards) - rewards = rewards / np.std(rewards - np.mean(rewards)) - gradients *= rewards - X = np.squeeze(np.vstack([self.states])) - Y = self.probs + self.learning_rate * np.squeeze(np.vstack([gradients])) - self.model.train_on_batch(X, Y) - self.states, self.probs, self.gradients, self.rewards = [], [], [], [] - - def load(self, name): - self.model.load_weights(name) - - def save(self, name): - self.model.save_weights(name) - -def preprocess(I): - I = I[35:195] - I = I[::2, ::2, 0] - I[I == 144] = 0 - I[I == 109] = 0 - I[I != 0] = 1 - return I.astype(np.float).ravel() - -if __name__ == "__main__": - env = gym.make("Pong-v0") - state = env.reset() - prev_x = None - score = 0 - episode = 0 - - state_size = 80 * 80 - action_size = env.action_space.n - agent = PGAgent(state_size, action_size) - agent.load('./save_model/pong_reinforce.h5') - while True: - env.render() - - cur_x = preprocess(state) - x = cur_x - prev_x if prev_x is not None else np.zeros(state_size) - prev_x = cur_x - - action, prob = agent.act(x) - state, reward, done, info = env.step(action) - score += reward - agent.remember(x, action, prob, reward) - - if done: - episode += 1 - agent.train() - print('Episode: %d - Score: %f.' % (episode, score)) - score = 0 - state = env.reset() - prev_x = None - if episode > 1 and episode % 10 == 0: - agent.save('./save_model/pong_reinforce.h5') diff --git a/3-atari/2-pong/save_model/pong_reinforce.h5 b/3-atari/2-pong/save_model/pong_reinforce.h5 deleted file mode 100644 index 6e0f2a6f..00000000 Binary files a/3-atari/2-pong/save_model/pong_reinforce.h5 and /dev/null differ diff --git a/3-atari/2-ppo.py b/3-atari/2-ppo.py new file mode 100644 index 00000000..90828584 --- /dev/null +++ b/3-atari/2-ppo.py @@ -0,0 +1,239 @@ +"""PPO agent for Atari (Breakout / Pong). + +Schulman et al., 2017: "Proximal Policy Optimization Algorithms" +(arXiv:1707.06347). Same clipped-surrogate + GAE objective as the +cartpole PPO, but with the Nature CNN as the shared trunk and the +DeepMind reward clipping that keeps the value function stable. + +Rollout uses 8 parallel envs via SyncVectorEnv (CleanRL convention). +Bump TOTAL_FRAMES well past the default for paper-quality results. +""" +import numpy as np +import torch +import torch.nn as nn +import torch.optim as optim + +from env import make_env, make_vec_env, parse_args, pick_device, run_test_loop + + +SAVE_PATH = "atari_ppo.pt" +TOTAL_FRAMES = 10_000_000 +N_ENVS = 8 +ROLLOUT_STEPS = 128 # batch = N_ENVS * ROLLOUT_STEPS = 1024 +EPOCHS = 4 +MINIBATCH_SIZE = 256 +CLIP_COEF = 0.1 +GAMMA = 0.99 +GAE_LAMBDA = 0.95 +LR = 2.5e-4 +VALUE_COEF = 0.5 +ENTROPY_COEF = 0.01 +MAX_GRAD_NORM = 0.5 + + +def _ortho(layer, gain): + nn.init.orthogonal_(layer.weight, gain) + nn.init.zeros_(layer.bias) + return layer + + +# Nature CNN shared trunk + policy and value heads. +class ActorCritic(nn.Module): + def __init__(self, n_actions): + super().__init__() + self.conv = nn.Sequential( + _ortho(nn.Conv2d(4, 32, kernel_size=8, stride=4), 2 ** 0.5), nn.ReLU(), + _ortho(nn.Conv2d(32, 64, kernel_size=4, stride=2), 2 ** 0.5), nn.ReLU(), + _ortho(nn.Conv2d(64, 64, kernel_size=3, stride=1), 2 ** 0.5), nn.ReLU(), + nn.Flatten(), + _ortho(nn.Linear(64 * 7 * 7, 512), 2 ** 0.5), nn.ReLU(), + ) + # gain=0.01 keeps the initial action distribution close to uniform. + self.policy = _ortho(nn.Linear(512, n_actions), 0.01) + self.value = _ortho(nn.Linear(512, 1), 1.0) + + def forward(self, x): + h = self.conv(x.float() / 255.0) + return self.policy(h), self.value(h).squeeze(-1) + + +def compute_gae(rewards, values, dones, last_value): + advantages = np.zeros_like(rewards, dtype=np.float32) + gae = 0.0 + for t in reversed(range(len(rewards))): + next_v = last_value if t == len(rewards) - 1 else values[t + 1] + next_nonterminal = 1.0 - dones[t] + delta = rewards[t] + GAMMA * next_v * next_nonterminal - values[t] + gae = delta + GAMMA * GAE_LAMBDA * next_nonterminal * gae + advantages[t] = gae + returns = advantages + values + return advantages, returns + + +if __name__ == "__main__": + args = parse_args() + device = pick_device(args.device) + + if args.test: + env = make_env(args) + n_actions = env.action_space.n + model = ActorCritic(n_actions).to(device) + model.load_state_dict(torch.load(SAVE_PATH, map_location=device)) + def policy_action(obs): + with torch.no_grad(): + t = torch.as_tensor(np.asarray(obs), device=device).unsqueeze(0) + logits, _ = model(t) + return int(torch.distributions.Categorical(logits=logits).sample().item()) + run_test_loop(env, policy_action) + + envs = make_vec_env(args, N_ENVS) + n_actions = envs.single_action_space.n + obs_shape = envs.single_observation_space.shape # (4, 84, 84) + + model = ActorCritic(n_actions).to(device) + optimizer = optim.Adam(model.parameters(), lr=LR, eps=1e-5) + + if args.wandb: + import wandb + wandb.init(project="rl-atari-ppo", config={ + "env": args.env, "n_envs": N_ENVS, "rollout_steps": ROLLOUT_STEPS, + "total_frames": TOTAL_FRAMES, "epochs": EPOCHS, + "minibatch_size": MINIBATCH_SIZE, "clip_coef": CLIP_COEF, + "gamma": GAMMA, "gae_lambda": GAE_LAMBDA, "lr": LR, + "value_coef": VALUE_COEF, "entropy_coef": ENTROPY_COEF, + }) + + print(f"device: {device}, env: {args.env}, actions: {n_actions}, n_envs: {N_ENVS}") + + batch_size = ROLLOUT_STEPS * N_ENVS + frames_per_update = batch_size + n_updates = TOTAL_FRAMES // frames_per_update + obs, _ = envs.reset() + ep_returns_per_env = np.zeros(N_ENVS, dtype=np.float32) # per-life (resets every life loss) + game_returns_per_env = np.zeros(N_ENVS, dtype=np.float32) # per-game (resets only on real game-over) + ep_returns = [] + game_returns = [] + + for update in range(1, n_updates + 1): + # Linear LR anneal from LR -> 0 over the run (CleanRL convention). + lr_now = LR * (1.0 - (update - 1) / n_updates) + for g in optimizer.param_groups: + g["lr"] = lr_now + + obs_buf = np.zeros((ROLLOUT_STEPS, N_ENVS, *obs_shape), dtype=np.uint8) + act_buf = np.zeros((ROLLOUT_STEPS, N_ENVS), dtype=np.int64) + logp_buf = np.zeros((ROLLOUT_STEPS, N_ENVS), dtype=np.float32) + rew_buf = np.zeros((ROLLOUT_STEPS, N_ENVS), dtype=np.float32) + done_buf = np.zeros((ROLLOUT_STEPS, N_ENVS), dtype=np.float32) + val_buf = np.zeros((ROLLOUT_STEPS, N_ENVS), dtype=np.float32) + + # --- Rollout --- + for t in range(ROLLOUT_STEPS): + with torch.no_grad(): + obs_t = torch.as_tensor(np.asarray(obs), device=device) + logits, value = model(obs_t) + dist = torch.distributions.Categorical(logits=logits) + action = dist.sample() + logp = dist.log_prob(action) + + obs_buf[t] = np.asarray(obs) + act_buf[t] = action.cpu().numpy() + logp_buf[t] = logp.cpu().numpy() + val_buf[t] = value.cpu().numpy() + + next_obs, reward, terminated, truncated, info = envs.step(act_buf[t]) + done = np.logical_or(terminated, truncated) + ep_returns_per_env += reward + game_returns_per_env += reward + rew_buf[t] = np.sign(reward).astype(np.float32) # DeepMind reward clipping + done_buf[t] = done.astype(np.float32) + + # LifeLossTerminalEnv tags each step's info with game_over (True only on real game-over). + game_over = info.get("game_over", done) + for i in range(N_ENVS): + if done[i]: + ep_returns.append(float(ep_returns_per_env[i])) + ep_returns_per_env[i] = 0.0 + if bool(game_over[i]): + game_returns.append(float(game_returns_per_env[i])) + game_returns_per_env[i] = 0.0 + obs = next_obs + + # --- GAE --- + with torch.no_grad(): + obs_t = torch.as_tensor(np.asarray(obs), device=device) + _, last_value = model(obs_t) + advantages, returns = compute_gae(rew_buf, val_buf, done_buf, last_value.cpu().numpy()) + + # Flatten (T, N_ENVS, ...) -> (T*N_ENVS, ...) + obs_t = torch.as_tensor(obs_buf.reshape(batch_size, *obs_shape), device=device) + act_t = torch.as_tensor(act_buf.reshape(batch_size), device=device) + old_logp_t = torch.as_tensor(logp_buf.reshape(batch_size), device=device) + old_val_t = torch.as_tensor(val_buf.reshape(batch_size), device=device) + adv_t = torch.as_tensor(advantages.reshape(batch_size), device=device) + ret_t = torch.as_tensor(returns.reshape(batch_size), device=device) + + # --- PPO updates --- + idx = np.arange(batch_size) + pl_sum = vl_sum = ent_sum = 0.0 + n_mb = 0 + for _ in range(EPOCHS): + np.random.shuffle(idx) + for start in range(0, batch_size, MINIBATCH_SIZE): + mb = idx[start:start + MINIBATCH_SIZE] + logits, values = model(obs_t[mb]) + dist = torch.distributions.Categorical(logits=logits) + new_logp = dist.log_prob(act_t[mb]) + entropy = dist.entropy().mean() + + # Advantage normalization per minibatch (CleanRL convention). + mb_adv = adv_t[mb] + mb_adv = (mb_adv - mb_adv.mean()) / (mb_adv.std() + 1e-8) + + ratio = (new_logp - old_logp_t[mb]).exp() + unclipped = ratio * mb_adv + clipped = torch.clamp(ratio, 1 - CLIP_COEF, 1 + CLIP_COEF) * mb_adv + policy_loss = -torch.min(unclipped, clipped).mean() + + # Value loss with clipping around the old value prediction. + v_clipped = old_val_t[mb] + torch.clamp( + values - old_val_t[mb], -CLIP_COEF, CLIP_COEF) + vl_unclipped = (values - ret_t[mb]).pow(2) + vl_clipped = (v_clipped - ret_t[mb]).pow(2) + value_loss = 0.5 * torch.max(vl_unclipped, vl_clipped).mean() + + loss = policy_loss + VALUE_COEF * value_loss - ENTROPY_COEF * entropy + + optimizer.zero_grad() + loss.backward() + nn.utils.clip_grad_norm_(model.parameters(), MAX_GRAD_NORM) + optimizer.step() + + pl_sum += policy_loss.item() + vl_sum += value_loss.item() + ent_sum += entropy.item() + n_mb += 1 + + global_step = update * frames_per_update + if ep_returns: + life_mean = float(np.mean(ep_returns[-20:])) + game_mean = float(np.mean(game_returns[-20:])) if game_returns else 0.0 + print(f"update: {update:>4} frames: {global_step:>8} " + f"per_life: {life_mean:.1f} per_game: {game_mean:.1f} " + f"lives: {len(ep_returns)} games: {len(game_returns)}") + if args.wandb: + log = { + "global_step": global_step, + "policy_loss": pl_sum / n_mb, + "value_loss": vl_sum / n_mb, + "entropy": ent_sum / n_mb, + "lr": lr_now, + } + if ep_returns: + log["recent_mean_return"] = float(np.mean(ep_returns[-20:])) + if game_returns: + log["recent_mean_game_return"] = float(np.mean(game_returns[-20:])) + wandb.log(log, step=global_step) + + torch.save(model.state_dict(), SAVE_PATH) + print(f"Saved trained model to {SAVE_PATH}") diff --git a/3-atari/LICENSE b/3-atari/LICENSE deleted file mode 100644 index 5c61d8af..00000000 --- a/3-atari/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2017 Keon Kim - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/3-atari/env.py b/3-atari/env.py new file mode 100644 index 00000000..1fb8513e --- /dev/null +++ b/3-atari/env.py @@ -0,0 +1,148 @@ +"""Shared Atari setup for DQN and PPO. + +Picks Breakout or Pong via --env, applies the standard Atari preprocessing +(frameskip 4, 84x84 grayscale, framestack 4), and exposes the same +--render / --test CLI as the cartpole scripts. + +Default device picks CUDA, falls back to MPS (Apple Silicon), then CPU. +""" +import argparse +import sys + +import ale_py +import gymnasium as gym +import numpy as np +import pygame +import torch + +gym.register_envs(ale_py) + + +# Breakout (and a few other games) require pressing FIRE to launch the ball +# after each reset / life loss. AtariPreprocessing only does NOOPs, so without +# this the agent wastes a lot of frames waiting for a random FIRE. +class FireResetEnv(gym.Wrapper): + def reset(self, **kwargs): + self.env.reset(**kwargs) + obs, _, terminated, truncated, _ = self.env.step(1) # FIRE + if terminated or truncated: + obs, _ = self.env.reset(**kwargs) + return obs, {} + + +# Treats each life as its own episode for bootstrapping (so Q-targets / GAE don't +# value-chain across deaths) but only resets the real game when all lives are +# gone. Without this, every life loss triggers a full env.reset() — burning +# frames on noop_max + FIRE and breaking long-horizon credit assignment. +class LifeLossTerminalEnv(gym.Wrapper): + def __init__(self, env): + super().__init__(env) + self.lives = 0 + self.game_over = True + + def step(self, action): + obs, reward, terminated, truncated, info = self.env.step(action) + self.game_over = terminated or truncated + lives = info.get("lives", 0) + if 0 < lives < self.lives: + terminated = True + self.lives = lives + info["game_over"] = self.game_over + return obs, reward, terminated, truncated, info + + def reset(self, **kwargs): + if self.game_over: + obs, info = self.env.reset(**kwargs) + else: + # Fake terminal from a life loss — advance one frame instead of + # resetting so the game keeps its remaining lives. + obs, _, terminated, truncated, info = self.env.step(0) + if terminated or truncated: + obs, info = self.env.reset(**kwargs) + self.lives = info.get("lives", 0) + return obs, info + +ENV_IDS = { + "breakout": "ALE/Breakout-v5", + "pong": "ALE/Pong-v5", +} + + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument("--env", choices=list(ENV_IDS), default="breakout", + help="which Atari game to train on") + p.add_argument("--render", action="store_true", + help="open a window during training (much slower)") + p.add_argument("--test", action="store_true", + help="load the saved checkpoint and just play (no learning)") + p.add_argument("--device", choices=["auto", "cpu", "cuda", "mps"], default="auto", + help="override the auto-selected torch device") + p.add_argument("--wandb", action="store_true", + help="log metrics to Weights & Biases") + return p.parse_args() + + +def make_env(args): + """Create an Atari env with the standard preprocessing pipeline.""" + env_id = ENV_IDS[args.env] + # frameskip=1 here because AtariPreprocessing applies its own. + env = gym.make(env_id, frameskip=1, + render_mode="human" if (args.render or args.test) else None) + env = gym.wrappers.AtariPreprocessing( + env, + noop_max=30, + frame_skip=4, + screen_size=84, + terminal_on_life_loss=False, # handled by LifeLossTerminalEnv below + grayscale_obs=True, + scale_obs=False, # keep uint8; we normalize in the model + ) + if "FIRE" in env.unwrapped.get_action_meanings(): + env = FireResetEnv(env) + env = LifeLossTerminalEnv(env) + env = gym.wrappers.FrameStackObservation(env, stack_size=4) + return env + + +def make_vec_env(args, n_envs): + """Bundle n_envs copies of make_env into a SyncVectorEnv.""" + return gym.vector.SyncVectorEnv([lambda: make_env(args) for _ in range(n_envs)]) + + +def pick_device(arg="auto"): + if arg != "auto": + return torch.device(arg) + if torch.cuda.is_available(): + return torch.device("cuda") + if torch.backends.mps.is_available(): + return torch.device("mps") + return torch.device("cpu") + + +def quit_if_window_closed(env): + """Exit cleanly when the user clicks the window's X. + + No-op on headless runs (no pygame display initialized). + """ + if not pygame.display.get_init(): + return + for event in pygame.event.get(): + if event.type == pygame.QUIT: + env.close() + sys.exit() + + +def run_test_loop(env, get_action): + """Replay episodes forever using the supplied action picker.""" + while True: + obs, _ = env.reset() + done = False + score = 0.0 + while not done: + quit_if_window_closed(env) + action = get_action(np.asarray(obs)) + obs, reward, terminated, truncated, _ = env.step(action) + done = terminated or truncated + score += reward + print(f"test score: {score}") diff --git a/4-atari-hard/1-ppo-rnd.py b/4-atari-hard/1-ppo-rnd.py new file mode 100644 index 00000000..4521046d --- /dev/null +++ b/4-atari-hard/1-ppo-rnd.py @@ -0,0 +1,467 @@ +"""PPO + RND (Random Network Distillation) for hard-exploration Atari. + +Burda et al., 2018: "Exploration by Random Network Distillation" +(arXiv:1810.12894). Vanilla PPO scores 0 on Montezuma's Revenge because the +first reward needs a ~100-step specific action sequence. RND adds an +intrinsic curiosity bonus that pulls the agent toward novel states: + + target_net (frozen random CNN) : s -> f_target(s) + predictor_net (learned) : s -> f_pred(s) + intrinsic_reward(s) = || f_pred(s) - f_target(s) ||^2 + +Novel states have high prediction error (predictor never saw them); seen +states drop to near zero. Five things make RND actually work: + + 1. RND input is the SINGLE last frame, not the 4-stack (avoids + overfitting to stack correlations). + 2. That input is normalized with running mean/std and clipped to + [-5, 5]; the stats are seeded by 50 rollouts of a random agent + BEFORE training starts. + 3. The intrinsic stream uses two separate value heads and its own GAE + that is NON-EPISODIC (next_nonterminal = 1 always) so curiosity can + chain across deaths. The paper calls this the most impactful design + choice. + 4. Intrinsic rewards are divided by the running std of their discounted + returns -- scale only, no mean centering. + 5. The predictor is updated on only ~25% of each minibatch so it + doesn't converge fast and kill the bonus. + +Combined advantage: A = ext_coef * A_ext + int_coef * A_int. +""" +import numpy as np +import torch +import torch.nn as nn +import torch.optim as optim + +from env import make_vec_env, parse_args, pick_device, seed_all, RunLogger + + +SAVE_PATH = "atari_ppo_rnd.pt" +TOTAL_FRAMES = 10_000_000 +N_ENVS = 128 # envpool: max trajectory diversity; swap-heavy on 8 GB +ROLLOUT_STEPS = 128 # batch = N_ENVS * ROLLOUT_STEPS = 16384 +EPOCHS = 4 +MINIBATCH_SIZE = 2048 # 8 minibatches per epoch at batch=16384 +CLIP_COEF = 0.1 +GAMMA_EXT = 0.999 # sparse-reward games need long horizons +GAMMA_INT = 0.99 # curiosity is short-horizon by nature +GAE_LAMBDA = 0.95 +LR = 1e-4 +EXT_COEF = 2.0 +INT_COEF = 1.0 +VALUE_COEF = 0.5 +ENTROPY_COEF = 0.01 # paper uses 0.001 (assumes 1B+ frames); at 10M frames the + # policy entropy collapsed by ~50% before any breakthrough, + # so we use the standard PPO value to keep exploration alive +MAX_GRAD_NORM = 0.5 +PREDICTOR_UPDATE_PROPORTION = 0.05 # paper default 0.25 saturates the predictor in ~500k + # frames at our scale, killing the intrinsic signal; slow it + # down 5x so novelty stays alive long enough to matter +OBS_NORM_WARMUP_ROLLOUTS = 50 # 50 * ROLLOUT_STEPS random transitions before training + + +def _ortho(layer, gain): + nn.init.orthogonal_(layer.weight, gain) + nn.init.zeros_(layer.bias) + return layer + + +# Same Nature CNN trunk as 3-atari/2-ppo.py, but with two value heads. +class ActorCriticRND(nn.Module): + def __init__(self, n_actions): + super().__init__() + self.conv = nn.Sequential( + _ortho(nn.Conv2d(4, 32, kernel_size=8, stride=4), 2 ** 0.5), nn.ReLU(), + _ortho(nn.Conv2d(32, 64, kernel_size=4, stride=2), 2 ** 0.5), nn.ReLU(), + _ortho(nn.Conv2d(64, 64, kernel_size=3, stride=1), 2 ** 0.5), nn.ReLU(), + nn.Flatten(), + _ortho(nn.Linear(64 * 7 * 7, 512), 2 ** 0.5), nn.ReLU(), + ) + self.policy = _ortho(nn.Linear(512, n_actions), 0.01) + self.value_ext = _ortho(nn.Linear(512, 1), 1.0) + self.value_int = _ortho(nn.Linear(512, 1), 1.0) # second head: intrinsic returns + + def forward(self, x): + h = self.conv(x.float() / 255.0) + return self.policy(h), self.value_ext(h).squeeze(-1), self.value_int(h).squeeze(-1) + + +# RND target/predictor share the same conv backbone with LeakyReLU +# (paper section 4.1). Input is a SINGLE 84x84 frame normalized to ~[-5, 5]. +def _rnd_conv(): + return nn.Sequential( + nn.Conv2d(1, 32, kernel_size=8, stride=4), nn.LeakyReLU(), + nn.Conv2d(32, 64, kernel_size=4, stride=2), nn.LeakyReLU(), + nn.Conv2d(64, 64, kernel_size=3, stride=1), nn.LeakyReLU(), + nn.Flatten(), + ) + + +class RNDTarget(nn.Module): + def __init__(self): + super().__init__() + self.conv = _rnd_conv() + self.fc = nn.Linear(64 * 7 * 7, 512) + + def forward(self, x): + return self.fc(self.conv(x)) + + +class RNDPredictor(nn.Module): + """Slightly deeper than target — two extra ReLU FCs so it has the + capacity to actually fit the target's random projection.""" + def __init__(self): + super().__init__() + self.conv = _rnd_conv() + self.head = nn.Sequential( + nn.Linear(64 * 7 * 7, 512), nn.ReLU(), + nn.Linear(512, 512), nn.ReLU(), + nn.Linear(512, 512), + ) + + def forward(self, x): + return self.head(self.conv(x)) + + +class RunningMeanStd: + """Welford / Chan parallel algorithm. Used for both obs (last frame) + and intrinsic-return scaling.""" + def __init__(self, shape=()): + self.mean = np.zeros(shape, dtype=np.float64) + self.var = np.ones(shape, dtype=np.float64) + self.count = 1e-4 + + def update(self, batch): + bm = batch.mean(axis=0) + bv = batch.var(axis=0) + bc = batch.shape[0] + delta = bm - self.mean + tot = self.count + bc + new_mean = self.mean + delta * bc / tot + m_a = self.var * self.count + m_b = bv * bc + M2 = m_a + m_b + delta ** 2 * self.count * bc / tot + self.mean = new_mean + self.var = M2 / tot + self.count = tot + + +def normalize_obs_for_rnd(frame, obs_rms): + """frame: (..., 84, 84) uint8 or float. Return float32 (..., 1, 84, 84) + centered/scaled by obs_rms and clipped to [-5, 5] per paper.""" + x = frame.astype(np.float32) + x = (x - obs_rms.mean) / np.sqrt(obs_rms.var + 1e-8) + x = np.clip(x, -5.0, 5.0) + return x.astype(np.float32) + + +def compute_gae(rewards, values, nonterminals, last_value, gamma, lam): + """Generic GAE. Pass nonterminals=1-dones for the extrinsic (episodic) + stream, or all-ones for the intrinsic (non-episodic) stream.""" + advantages = np.zeros_like(rewards, dtype=np.float32) + gae = 0.0 + for t in reversed(range(len(rewards))): + next_v = last_value if t == len(rewards) - 1 else values[t + 1] + delta = rewards[t] + gamma * next_v * nonterminals[t] - values[t] + gae = delta + gamma * lam * nonterminals[t] * gae + advantages[t] = gae + return advantages, advantages + values + + +def warmup_obs_rms(envs, obs_rms, n_steps): + """Step a random agent so obs running stats are realistic before training. + Without this, the first intrinsic rewards are wildly scaled and the + predictor never recovers. + + Updates obs_rms incrementally each step to avoid building a multi-GB list + of frames when N_ENVS is large.""" + print(f"warmup: stepping random agent for {n_steps} env steps to seed obs RMS...") + obs, _ = envs.reset() + n_actions = envs.action_space.n + for _ in range(n_steps): + actions = np.random.randint(0, n_actions, size=envs.num_envs, dtype=np.int32) + next_obs, _, _, _, _ = envs.step(actions) + # FrameStackObservation gives (n_envs, 4, 84, 84); update RMS with newest frames only. + obs_rms.update(np.asarray(next_obs)[:, -1, :, :]) + obs = next_obs + print(f" obs_rms seeded with {obs_rms.count:.0f} samples, " + f"mean={obs_rms.mean.mean():.2f}, std={np.sqrt(obs_rms.var).mean():.2f}") + return obs + + +if __name__ == "__main__": + args = parse_args() + # CLI overrides for the in-file constants (omit to keep defaults) + if args.n_envs: + N_ENVS = args.n_envs + if args.total_frames: + TOTAL_FRAMES = args.total_frames + if args.seed is not None: + seed_all(args.seed) + device = pick_device(args.device) + envs = make_vec_env(args, N_ENVS, seed=args.seed or 0) + n_actions = envs.action_space.n + obs_shape = envs.observation_space.shape # (4, 84, 84) — envpool single-env spec + + model = ActorCriticRND(n_actions).to(device) + rnd_target = RNDTarget().to(device) + rnd_predictor = RNDPredictor().to(device) + for p in rnd_target.parameters(): + p.requires_grad_(False) + rnd_target.eval() + + optimizer = optim.Adam( + list(model.parameters()) + list(rnd_predictor.parameters()), + lr=LR, eps=1e-5, + ) + + # Optional run-directory outputs (metrics.jsonl, checkpoints, resume). + # Inert without --run-dir, so this script still runs standalone. + logger = RunLogger(args.run_dir, args.ckpt_every) + start_update = 0 # updated on resume + + if args.wandb: + import wandb + wandb.init(project="rl-atari-hard-ppo-rnd", config={ + "env": args.env, "n_envs": N_ENVS, "rollout_steps": ROLLOUT_STEPS, + "total_frames": TOTAL_FRAMES, "epochs": EPOCHS, + "minibatch_size": MINIBATCH_SIZE, "clip_coef": CLIP_COEF, + "gamma_ext": GAMMA_EXT, "gamma_int": GAMMA_INT, "gae_lambda": GAE_LAMBDA, + "lr": LR, "ext_coef": EXT_COEF, "int_coef": INT_COEF, + "value_coef": VALUE_COEF, "entropy_coef": ENTROPY_COEF, + "predictor_update_proportion": PREDICTOR_UPDATE_PROPORTION, + "obs_norm_warmup_rollouts": OBS_NORM_WARMUP_ROLLOUTS, + }) + + print(f"device: {device}, env: {args.env}, actions: {n_actions}, n_envs: {N_ENVS}") + + obs_rms = RunningMeanStd(shape=(84, 84)) + int_ret_rms = RunningMeanStd(shape=()) + + obs = warmup_obs_rms(envs, obs_rms, n_steps=OBS_NORM_WARMUP_ROLLOUTS * ROLLOUT_STEPS) + + batch_size = ROLLOUT_STEPS * N_ENVS + n_updates = TOTAL_FRAMES // batch_size + ep_returns_per_env = np.zeros(N_ENVS, dtype=np.float32) + ep_returns = [] # extrinsic (raw, unclipped) per-game returns + int_filter = np.zeros(N_ENVS, dtype=np.float64) # discounted intrinsic returns for RMS + + def _state_fn(): + """Full checkpoint state — normalizers / int_filter / update too, so resume is exact.""" + return { + "actor_critic": model.state_dict(), + "rnd_predictor": rnd_predictor.state_dict(), + "rnd_target": rnd_target.state_dict(), + "optimizer": optimizer.state_dict(), + "obs_rms": {"mean": obs_rms.mean, "var": obs_rms.var, "count": obs_rms.count}, + "int_ret_rms": {"mean": int_ret_rms.mean, "var": int_ret_rms.var, "count": int_ret_rms.count}, + "int_filter": int_filter, "update": update, + "ep_returns": ep_returns[-200:], + } + + # --- resume (restores normalizers / optimizer / update counter) --- + resume_path = logger.resolve_resume(args.resume) + if resume_path: + ckpt = torch.load(resume_path, map_location=device, weights_only=False) + model.load_state_dict(ckpt["actor_critic"]) + rnd_predictor.load_state_dict(ckpt["rnd_predictor"]) + rnd_target.load_state_dict(ckpt["rnd_target"]) + optimizer.load_state_dict(ckpt["optimizer"]) + for rms, saved in [(obs_rms, ckpt["obs_rms"]), (int_ret_rms, ckpt["int_ret_rms"])]: + rms.mean, rms.var, rms.count = saved["mean"], saved["var"], saved["count"] + int_filter = ckpt["int_filter"] + ep_returns = list(ckpt["ep_returns"]) + start_update = ckpt["update"] + print(f"resumed from {resume_path} at update {start_update}") + + global_step = start_update * batch_size # defined up front so finalize() has a value + for update in range(start_update + 1, n_updates + 1): + lr_now = LR * (1.0 - (update - 1) / n_updates) + for g in optimizer.param_groups: + g["lr"] = lr_now + + obs_buf = np.zeros((ROLLOUT_STEPS, N_ENVS, *obs_shape), dtype=np.uint8) + act_buf = np.zeros((ROLLOUT_STEPS, N_ENVS), dtype=np.int64) + logp_buf = np.zeros((ROLLOUT_STEPS, N_ENVS), dtype=np.float32) + rew_ext_buf = np.zeros((ROLLOUT_STEPS, N_ENVS), dtype=np.float32) + rew_int_buf = np.zeros((ROLLOUT_STEPS, N_ENVS), dtype=np.float32) + val_ext_buf = np.zeros((ROLLOUT_STEPS, N_ENVS), dtype=np.float32) + val_int_buf = np.zeros((ROLLOUT_STEPS, N_ENVS), dtype=np.float32) + done_buf = np.zeros((ROLLOUT_STEPS, N_ENVS), dtype=np.float32) + + # --- Rollout --- + for t in range(ROLLOUT_STEPS): + with torch.no_grad(): + obs_t = torch.as_tensor(np.asarray(obs), device=device) + logits, v_ext, v_int = model(obs_t) + dist = torch.distributions.Categorical(logits=logits) + action = dist.sample() + logp = dist.log_prob(action) + + obs_buf[t] = np.asarray(obs) + act_buf[t] = action.cpu().numpy() + logp_buf[t] = logp.cpu().numpy() + val_ext_buf[t] = v_ext.cpu().numpy() + val_int_buf[t] = v_int.cpu().numpy() + + next_obs, reward, terminated, truncated, _ = envs.step(act_buf[t].astype(np.int32)) + done = np.logical_or(terminated, truncated) + ep_returns_per_env += reward + rew_ext_buf[t] = np.sign(reward).astype(np.float32) + done_buf[t] = done.astype(np.float32) + + # Intrinsic reward: predict on the next frame's last channel. + next_last = np.asarray(next_obs)[:, -1, :, :] + obs_rms.update(next_last) + with torch.no_grad(): + x = normalize_obs_for_rnd(next_last, obs_rms) # (n_envs, 84, 84) + x_t = torch.as_tensor(x, device=device).unsqueeze(1) # (n_envs, 1, 84, 84) + err = (rnd_predictor(x_t) - rnd_target(x_t)).pow(2).mean(dim=-1) + rew_int_buf[t] = err.cpu().numpy() + + for i in range(N_ENVS): + if done[i]: + ep_returns.append(float(ep_returns_per_env[i])) + ep_returns_per_env[i] = 0.0 + obs = next_obs + + # --- Intrinsic reward normalization (running std of discounted intrinsic returns) --- + # Walk forward through the rollout, accumulating per-env discounted returns, + # update int_ret_rms with all visited values, then scale rew_int by current std. + for t in range(ROLLOUT_STEPS): + int_filter = int_filter * GAMMA_INT + rew_int_buf[t] + int_ret_rms.update(int_filter.copy()) + rew_int_buf = rew_int_buf / np.sqrt(int_ret_rms.var + 1e-8) + + # --- Dual GAE: extrinsic episodic, intrinsic non-episodic --- + with torch.no_grad(): + obs_t = torch.as_tensor(np.asarray(obs), device=device) + _, last_v_ext, last_v_int = model(obs_t) + adv_ext, ret_ext = compute_gae( + rew_ext_buf, val_ext_buf, 1.0 - done_buf, + last_v_ext.cpu().numpy(), GAMMA_EXT, GAE_LAMBDA, + ) + adv_int, ret_int = compute_gae( + rew_int_buf, val_int_buf, np.ones_like(done_buf), + last_v_int.cpu().numpy(), GAMMA_INT, GAE_LAMBDA, + ) + adv_combined = EXT_COEF * adv_ext + INT_COEF * adv_int + + # Flatten (T, N_ENVS, ...) -> (T*N_ENVS, ...) + obs_t = torch.as_tensor(obs_buf.reshape(batch_size, *obs_shape), device=device) + act_t = torch.as_tensor(act_buf.reshape(batch_size), device=device) + old_logp_t = torch.as_tensor(logp_buf.reshape(batch_size), device=device) + adv_t = torch.as_tensor(adv_combined.reshape(batch_size), device=device) + ret_ext_t = torch.as_tensor(ret_ext.reshape(batch_size), device=device) + ret_int_t = torch.as_tensor(ret_int.reshape(batch_size), device=device) + # RND predictor input: precompute normalized last frames once per rollout. + last_frames = obs_buf[:, :, -1, :, :].reshape(batch_size, 84, 84) + rnd_in_t = torch.as_tensor( + normalize_obs_for_rnd(last_frames, obs_rms), device=device, + ).unsqueeze(1) # (batch, 1, 84, 84) + + # --- PPO + RND updates --- + idx = np.arange(batch_size) + pl_sum = vl_sum = ent_sum = rnd_sum = kl_sum = 0.0 + n_mb = 0 + for _ in range(EPOCHS): + np.random.shuffle(idx) + for start in range(0, batch_size, MINIBATCH_SIZE): + mb = idx[start:start + MINIBATCH_SIZE] + logits, v_ext_pred, v_int_pred = model(obs_t[mb]) + dist = torch.distributions.Categorical(logits=logits) + new_logp = dist.log_prob(act_t[mb]) + entropy = dist.entropy().mean() + + mb_adv = adv_t[mb] + mb_adv = (mb_adv - mb_adv.mean()) / (mb_adv.std() + 1e-8) + + logratio = new_logp - old_logp_t[mb] + ratio = logratio.exp() + with torch.no_grad(): # approx KL, for diagnostics / logging + kl_sum += ((ratio - 1) - logratio).mean().item() + unclipped = ratio * mb_adv + clipped = torch.clamp(ratio, 1 - CLIP_COEF, 1 + CLIP_COEF) * mb_adv + policy_loss = -torch.min(unclipped, clipped).mean() + + v_ext_loss = (v_ext_pred - ret_ext_t[mb]).pow(2).mean() + v_int_loss = (v_int_pred - ret_int_t[mb]).pow(2).mean() + value_loss = 0.5 * (v_ext_loss + v_int_loss) + + # Predictor MSE on a random ~25% slice of the minibatch. + pred = rnd_predictor(rnd_in_t[mb]) + with torch.no_grad(): + tgt = rnd_target(rnd_in_t[mb]) + per_sample = (pred - tgt).pow(2).mean(dim=-1) + keep = (torch.rand_like(per_sample) < PREDICTOR_UPDATE_PROPORTION).float() + rnd_loss = (per_sample * keep).sum() / keep.sum().clamp(min=1.0) + + loss = (policy_loss + + VALUE_COEF * value_loss + - ENTROPY_COEF * entropy + + rnd_loss) + + optimizer.zero_grad() + loss.backward() + nn.utils.clip_grad_norm_( + list(model.parameters()) + list(rnd_predictor.parameters()), + MAX_GRAD_NORM, + ) + optimizer.step() + + pl_sum += policy_loss.item() + vl_sum += value_loss.item() + ent_sum += entropy.item() + rnd_sum += rnd_loss.item() + n_mb += 1 + + global_step = update * batch_size + if ep_returns: + recent = float(np.mean(ep_returns[-20:])) + print(f"update: {update:>4} frames: {global_step:>8} " + f"recent_mean_return: {recent:.1f} episodes: {len(ep_returns)} " + f"int_reward_mean: {rew_int_buf.mean():.3f} lr: {lr_now:.2e}") + + # structured log row + periodic / milestone / best checkpoints (no-ops without --run-dir) + loss_mean = (pl_sum + vl_sum + rnd_sum) / max(n_mb, 1) + gate = float(np.mean(ep_returns[-100:])) if ep_returns else None + logger.log(global_step, { + "game_return_mean_lastK": gate if gate is not None else 0.0, + "ep_return_mean": float(np.mean(ep_returns[-20:])) if ep_returns else 0.0, + "game_return_count": len(ep_returns), + "entropy": ent_sum / max(n_mb, 1), "approx_kl": kl_sum / max(n_mb, 1), + "policy_loss": pl_sum / max(n_mb, 1), "value_loss": vl_sum / max(n_mb, 1), + "predictor_loss": rnd_sum / max(n_mb, 1), + "int_rew_mean": float(rew_int_buf.mean()), "int_rew_std": float(rew_int_buf.std()), + "lr": lr_now, "nan_flag": int(not np.isfinite(loss_mean)), + }) + logger.checkpoint(global_step, _state_fn, gate=gate) + if args.wandb: + log = { + "global_step": global_step, + "policy_loss": pl_sum / n_mb, + "value_loss": vl_sum / n_mb, + "entropy": ent_sum / n_mb, + "rnd_loss": rnd_sum / n_mb, + "int_reward_mean": float(rew_int_buf.mean()), + "int_reward_std": float(rew_int_buf.std()), + "obs_rms_mean": float(obs_rms.mean.mean()), + "obs_rms_std": float(np.sqrt(obs_rms.var).mean()), + "int_ret_rms_std": float(np.sqrt(int_ret_rms.var)), + "lr": lr_now, + } + if ep_returns: + log["recent_mean_return"] = float(np.mean(ep_returns[-20:])) + wandb.log(log, step=global_step) + + # final checkpoint + final.json summary (no-ops without --run-dir), plus the + # legacy single-file save for `--test` replay. + logger.finalize(global_step, ep_returns, _state_fn, k=100) + torch.save({ + "actor_critic": model.state_dict(), + "rnd_predictor": rnd_predictor.state_dict(), + "rnd_target": rnd_target.state_dict(), + "obs_rms_mean": obs_rms.mean, + "obs_rms_var": obs_rms.var, + }, SAVE_PATH) + print(f"Saved trained model to {SAVE_PATH}") diff --git a/4-atari-hard/2-go-explore.py b/4-atari-hard/2-go-explore.py new file mode 100644 index 00000000..d9790e01 --- /dev/null +++ b/4-atari-hard/2-go-explore.py @@ -0,0 +1,438 @@ +"""Go-Explore Phase 1 (exploration phase) for Montezuma's Revenge. + +Ecoffet et al., 2019: "Go-Explore: a New Approach for Hard-Exploration +Problems" (arXiv:1901.10995); Nature 2021 version "First return, then +explore" (arXiv:2004.12919). No neural network: intrinsic-motivation methods +(RND etc.) suffer from detachment (forgetting promising frontiers) and +derailment (exploration noise breaking the return trip). Go-Explore fixes +both mechanically — remember everything in an archive, and RETURN exactly +via emulator state restore, then explore from there: + + archive: cell -> (best trajectory reaching it, emulator snapshot, score) + loop: sample cells (novelty-weighted) -> restore -> random exploration + -> add/update cells reached + +Design notes (verified against the official uber-research/go-explore code): + + 1. Cell key = grayscale frame -> cv2.resize to 11x8 (INTER_AREA) -> + quantize to 9 levels: floor(8 * p / 255). 88-byte key. + 2. Selection weight = 1 / sqrt(seen_times + 1) (Nature simplification); + sampling WITH replacement, batch of 100; the virtual DONE cell is + never selected. + 3. Exploration from a restored cell: up to K=100 agent steps, repeated + random actions (keep current action w.p. 0.95 -> geometric runs, + mean 20). Episode end = LIFE LOSS (or game over) -> the transition + maps to the DONE cell and the exploration episode aborts. + 4. Archive accept rule: replace/insert iff score is higher, or equal + score with a shorter trajectory. Scores are raw and unclipped. + On update the cell's counters reset and its snapshot/trajectory are + replaced; the *chosen* cell's chosen_since_new resets when anything + new is found. + 5. Trajectories are not stored per cell: a global append-only experience + log (prev_id linked list) + per-cell traj_last pointers reconstruct + any cell's action sequence — this is the demo source for a future + robustification phase, so the log is flushed to compressed chunks in + the run dir rather than discarded. + 6. ★ ALE pitfall (machine-verified): post-restore RAM/screen reads are + STALE until the next act. Cell keys come only from frames returned by + env.step(); the lives baseline travels in cell metadata. + 7. frames axis = agent steps actually EXECUTED by workers (frameskip 4 + applied; the hypothetical "replay from start" steps are not counted), + matching the harness budget/tier semantics. + 8. Phase-1 caveat: the score is a deterministic trajectory-search result, + NOT an RL-policy score. Never compare against sticky-action RL + numbers (e.g. the RND campaign) without this caveat. +""" +import multiprocessing as mp +import os +import pickle +import time + +import cv2 +import numpy as np + +from env_go_explore import ENV_IDS, RunLogger, make_restore_env, parse_args + + +TOTAL_FRAMES = 5_000_000 # agent steps executed (override with --total-frames) +BATCH_CELLS = 100 # cells sampled (with replacement) per iteration +EXPLORE_STEPS = 100 # K: max agent steps per exploration episode +ACTION_REPEAT_P = 0.95 # keep current action w.p. 0.95 (geometric, mean 20) +CELL_W, CELL_H = 11, 8 # downscale resolution (official fixed setting) +CELL_LEVELS = 8 # quantize to floor(8*p/255) -> values 0..8 +N_WORKERS = 12 # M4 Max 16 cores: leave headroom for master + OS +LOG_EVERY_BATCHES = 10 # metrics.jsonl cadence (~1M steps/100s at full speed) +EXPLOG_CHUNK = 1 << 22 # 4M entries per experience-log chunk (~40MB in RAM) +ROOM_RAM_BYTE = 3 # Montezuma current-room RAM index (diagnostic only) +DONE_KEY = (b"DONE", True) # virtual end-of-episode cell (never sampled) + + +def cell_key(frame): + """(210, 160) uint8 grayscale frame -> 88-byte archive key.""" + small = cv2.resize(frame, (CELL_W, CELL_H), interpolation=cv2.INTER_AREA) + return ((small / 255.0) * CELL_LEVELS).astype(np.uint8).tobytes() + + +class Cell: + """Archive entry. snapshot/lives describe the state AT this cell so a + worker can restore and keep exploring; traj_last points into the + experience log for trajectory reconstruction.""" + __slots__ = ("snapshot", "score", "traj_len", "traj_last", + "seen", "chosen", "chosen_since_new", "lives") + + def __init__(self, snapshot, score, traj_len, traj_last, lives): + self.snapshot = snapshot + self.score = score + self.traj_len = traj_len + self.traj_last = traj_last + self.lives = lives + self.seen = self.chosen = self.chosen_since_new = 0 + + +class ExperienceLog: + """Append-only step log as a prev_id linked list (design note 5). + + RAM holds only the active chunk; full chunks flush to + /chunk_NNNNN.npz (compressed — rewards/dones are almost all zero). + With dir=None (probes/tests) full chunks stay in RAM instead.""" + + def __init__(self, log_dir, chunk_size=EXPLOG_CHUNK, ancestor_dir=None): + self.dir = log_dir + if log_dir: + os.makedirs(log_dir, exist_ok=True) + self.chunk_size = chunk_size + self.ancestor = ancestor_dir # explog dir of the run we resumed FROM: + self.count = 0 # chunks flushed before the resume live there + self.n_flushed = 0 + self._ram_chunks = [] # dir=None mode only + self._cache = {} # chunk_idx -> loaded arrays (reconstruction) + self._new_chunk() + + def _new_chunk(self): + n = self.chunk_size + self.prev = np.empty(n, dtype=np.int64) + self.act = np.empty(n, dtype=np.uint8) + self.rew = np.empty(n, dtype=np.float32) + self.done = np.empty(n, dtype=np.uint8) + self.fill = 0 + + def append(self, prev_id, action, reward, done): + i = self.fill + self.prev[i], self.act[i], self.rew[i], self.done[i] = prev_id, action, reward, done + self.fill += 1 + idx = self.count + self.count += 1 + if self.fill == self.chunk_size: + self._flush() + return idx + + def _flush(self): + arrays = {"prev": self.prev[:self.fill], "act": self.act[:self.fill], + "rew": self.rew[:self.fill], "done": self.done[:self.fill]} + if self.dir: + tmp = os.path.join(self.dir, f"chunk_{self.n_flushed:05d}.tmp") + np.savez_compressed(tmp, **arrays) + os.replace(f"{tmp}.npz", os.path.join(self.dir, f"chunk_{self.n_flushed:05d}.npz")) + else: + self._ram_chunks.append({k: v.copy() for k, v in arrays.items()}) + self.n_flushed += 1 + self._new_chunk() + + def _chunk_path(self, chunk_idx): + """A flushed chunk lives in our own dir, or (after a cross-run-dir + resume) in the ancestor run's explog dir.""" + own = os.path.join(self.dir, f"chunk_{chunk_idx:05d}.npz") + if os.path.exists(own): + return own + if self.ancestor: + anc = os.path.join(self.ancestor, f"chunk_{chunk_idx:05d}.npz") + if os.path.exists(anc): + return anc + raise RuntimeError(f"explog chunk {chunk_idx} not found in {self.dir}" + + (f" or {self.ancestor}" if self.ancestor else "")) + + def _chunk(self, chunk_idx): + if chunk_idx == self.n_flushed: + return {"prev": self.prev, "act": self.act} + if self.dir: + if chunk_idx not in self._cache: + z = np.load(self._chunk_path(chunk_idx)) + self._cache[chunk_idx] = {"prev": z["prev"], "act": z["act"]} + return self._cache[chunk_idx] + return self._ram_chunks[chunk_idx] + + def reconstruct_actions(self, last_id): + """Walk the prev_id chain back to the root (-1); return actions in + forward order. This is how demos are rebuilt for replay/Phase 2.""" + actions = [] + idx = last_id + while idx >= 0: + c = self._chunk(idx // self.chunk_size) + off = idx % self.chunk_size + actions.append(int(c["act"][off])) + idx = int(c["prev"][off]) + return actions[::-1] + + def state(self): + return {"count": self.count, "n_flushed": self.n_flushed, + "chunk_size": self.chunk_size, + "cur_prev": self.prev[:self.fill].copy(), "cur_act": self.act[:self.fill].copy(), + "cur_rew": self.rew[:self.fill].copy(), "cur_done": self.done[:self.fill].copy()} + + def load_state(self, st): + assert st["chunk_size"] == self.chunk_size, "explog chunk_size mismatch" + if self.dir: # flushed chunks must be reachable (own dir or ancestor's) + self.n_flushed = st["n_flushed"] + for i in range(st["n_flushed"]): + self._chunk_path(i) # raises loudly if a chunk is missing + self.count, self.n_flushed = st["count"], st["n_flushed"] + self._new_chunk() + n = len(st["cur_prev"]) + self.prev[:n], self.act[:n] = st["cur_prev"], st["cur_act"] + self.rew[:n], self.done[:n] = st["cur_rew"], st["cur_done"] + self.fill = n + + +class Archive: + """Cell store + novelty-weighted selection + the accept rule. All updates + happen serially in the master process.""" + + def __init__(self): + self.cells = {} # (key_bytes, done_bool) -> Cell + self.rooms = set() # diagnostic only (RAM byte 3) + self.done_scores = [] # recent end-of-episode scores (logging) + + def seed_root(self, key, snapshot, lives): + self.cells[(key, False)] = Cell(snapshot, 0.0, 0, -1, lives) + + @property + def best_done_score(self): + c = self.cells.get(DONE_KEY) + return c.score if c else float("-inf") + + @property + def max_archive_score(self): + return max(c.score for k, c in self.cells.items() if k != DONE_KEY) + + def sample(self, n, rng): + """n cells with replacement, p ∝ 1/sqrt(seen+1); DONE excluded. + + Returns (key, CAPTURE) pairs that freeze the cell's snapshot/score/ + trajectory AT SAMPLING TIME. The trajectory walk must use the capture, + never the live cell: an earlier result in the same batch may replace + the cell, and stitching actions executed from the OLD state onto the + NEW prefix fabricates scores no single playthrough achieved + (2026-06-08 incident — caught by publish-time replay verification; + the official code ships these values inside the task for this reason).""" + keys = [k for k in self.cells if k != DONE_KEY] + w = np.array([1.0 / np.sqrt(self.cells[k].seen + 1.0) for k in keys]) + csum = np.cumsum(w) + picks = [] + for u in rng.random(n) * csum[-1]: + k = keys[min(int(np.searchsorted(csum, u)), len(keys) - 1)] + c = self.cells[k] + c.chosen += 1 + c.chosen_since_new += 1 + picks.append((k, {"snapshot": c.snapshot, "lives": c.lives, "score": c.score, + "traj_len": c.traj_len, "traj_last": c.traj_last})) + return picks + + def update_from_trajectory(self, chosen_key, capture, res, explog): + """Walk one exploration episode (master-side, serial): append to the + experience log, accumulate raw score from the SAMPLING-TIME capture + (never the live cell — see sample()), apply the accept rule (note 4).""" + chosen = self.cells.get(chosen_key) + cur_score = capture["score"] + cur_len = capture["traj_len"] + prev_id = capture["traj_last"] + found_new = False + seen_this_episode = set() + + for i in range(res["n_steps"]): + prev_id = explog.append(prev_id, res["actions"][i], res["rewards"][i], res["dones"][i]) + cur_score += float(res["rewards"][i]) + cur_len += 1 + done = bool(res["dones"][i]) + key = DONE_KEY if done else (res["keys"][i], False) + + cell = self.cells.get(key) + if cell is None: + self.cells[key] = Cell(res["snapshots"][i], cur_score, cur_len, prev_id, + res["lives"][i]) + self.cells[key].seen = 1 + seen_this_episode.add(key) + found_new = True + else: + if key not in seen_this_episode: + cell.seen += 1 + seen_this_episode.add(key) + if cur_score > cell.score or (cur_score == cell.score and cur_len < cell.traj_len): + cell.snapshot = res["snapshots"][i] + cell.score, cell.traj_len, cell.traj_last = cur_score, cur_len, prev_id + cell.lives = res["lives"][i] + cell.seen = cell.chosen = cell.chosen_since_new = 0 # reset_cell_on_update + found_new = True + if done: + self.done_scores.append(cur_score) + break + + if found_new and chosen is not None: + chosen.chosen_since_new = 0 + self.rooms.update(res["rooms"]) + + def state(self): + return {"cells": {k: {"snapshot": c.snapshot, "score": c.score, + "traj_len": c.traj_len, "traj_last": c.traj_last, + "seen": c.seen, "chosen": c.chosen, + "chosen_since_new": c.chosen_since_new, "lives": c.lives} + for k, c in self.cells.items()}, + "rooms": sorted(self.rooms), "done_scores": self.done_scores[-200:]} + + def load_state(self, st): + self.cells = {} + for k, d in st["cells"].items(): + c = Cell(d["snapshot"], d["score"], d["traj_len"], d["traj_last"], d["lives"]) + c.seen, c.chosen, c.chosen_since_new = d["seen"], d["chosen"], d["chosen_since_new"] + self.cells[k] = c + self.rooms = set(st["rooms"]) + self.done_scores = list(st["done_scores"]) + + +# --------------------------------------------------------------------------- +# Worker side. Top-level functions: mp 'spawn' re-imports this module, so the +# main body below stays behind the __main__ guard. Each worker owns one env. +# --------------------------------------------------------------------------- +_W = {} + + +def _worker_init(env_key): + _W["env"] = make_restore_env(env_key) + _W["ale"] = _W["env"].ale + + +def _explore_task(task): + """task = (snapshot bytes | None for root reset, lives, k, seed). + Restore -> up to k steps of repeated random actions; abort on life loss / + game over. Returns per-step arrays (keys/snapshots for archive insert).""" + snapshot, prev_lives, k, seed = task + env, ale = _W["env"], _W["ale"] + rng = np.random.default_rng(seed) + if snapshot is None: + env.reset(seed=0) + else: + ale.restoreState(pickle.loads(snapshot)) + # design note 6: NO reads here — the restored state is stale until we act + + n_actions = env.action_space.n + actions, rewards, dones, keys, snapshots, lives_list, rooms = [], [], [], [], [], [], set() + action = int(rng.integers(n_actions)) + for _ in range(k): + if rng.random() > ACTION_REPEAT_P: + action = int(rng.integers(n_actions)) + frame, reward, terminated, truncated, _ = env.step(action) + lives = ale.lives() + done = bool(terminated) or lives < prev_lives + actions.append(action) + rewards.append(float(reward)) + dones.append(done) + keys.append(cell_key(frame)) + snapshots.append(pickle.dumps(ale.cloneState())) + lives_list.append(lives) + rooms.add(int(ale.getRAM()[ROOM_RAM_BYTE])) + if done: + break + prev_lives = lives + return {"n_steps": len(actions), "actions": actions, "rewards": rewards, + "dones": dones, "keys": keys, "snapshots": snapshots, + "lives": lives_list, "rooms": rooms} + + +if __name__ == "__main__": + args = parse_args() + if args.total_frames: + TOTAL_FRAMES = args.total_frames + if args.n_workers: + N_WORKERS = args.n_workers + seed = args.seed if args.seed is not None else 0 + rng = np.random.default_rng(seed) + + logger = RunLogger(args.run_dir, args.ckpt_every) + explog_dir = os.path.join(args.run_dir, "explog") if args.run_dir else None + # cross-run-dir resume: flushed explog chunks live next to the checkpoint + # we resume from (the harness relaunches into a fresh run dir) + resume_path = logger.resolve_resume(args.resume) + ancestor = (os.path.join(os.path.dirname(os.path.dirname(resume_path)), "explog") + if resume_path else None) + explog = ExperienceLog(explog_dir, ancestor_dir=ancestor) + archive = Archive() + frames = 0 + batch = 0 + + def _state_fn(): + return {"version": 1, "frames": frames, "batch": batch, + "archive": archive.state(), "explog": explog.state(), + "rng": rng.bit_generator.state} + + # --- resume or seed the root cell --- + if resume_path: + import torch + ckpt = torch.load(resume_path, map_location="cpu", weights_only=False) + frames, batch = ckpt["frames"], ckpt["batch"] + archive.load_state(ckpt["archive"]) + explog.load_state(ckpt["explog"]) + rng.bit_generator.state = ckpt["rng"] + print(f"resumed from {resume_path}: frames={frames} batch={batch} " + f"cells={len(archive.cells)} explog={explog.count}") + else: + # root cell from a fresh reset (reset obs is NOT stale — note 6 only + # applies to restores) + env0 = make_restore_env(args.env) + frame0, _ = env0.reset(seed=0) + archive.seed_root(cell_key(np.asarray(frame0)), pickle.dumps(env0.ale.cloneState()), + env0.ale.lives()) + env0.close() + + print(f"env: {args.env} workers: {N_WORKERS} total_frames: {TOTAL_FRAMES:,} seed: {seed}") + ctx = mp.get_context("spawn") + t_start = time.time() + with ctx.Pool(N_WORKERS, initializer=_worker_init, initargs=(args.env,)) as pool: + while frames < TOTAL_FRAMES: + picks = archive.sample(BATCH_CELLS, rng) # (key, sampling-time capture) + tasks = [(cap["snapshot"], cap["lives"], EXPLORE_STEPS, int(rng.integers(2 ** 31))) + for _, cap in picks] + results = pool.map(_explore_task, tasks, chunksize=2) # ordered -> deterministic + for (key, cap), res in zip(picks, results): + archive.update_from_trajectory(key, cap, res, explog) + frames += res["n_steps"] + batch += 1 + + if batch % LOG_EVERY_BATCHES == 0: + best = archive.best_done_score + gate = best if best != float("-inf") else 0.0 + tail = archive.done_scores[-20:] + print(f"batch {batch:>6} frames {frames:>11,} cells {len(archive.cells):>6} " + f"best_done {gate:>8.0f} max_arch {archive.max_archive_score:>8.0f} " + f"rooms {len(archive.rooms):>3}") + logger.log(frames, { + "game_return_mean_lastK": gate, # semantics: best end-of-episode score (K=1) + "ep_return_mean": float(np.mean(tail)) if tail else 0.0, + "game_return_count": len(archive.done_scores), + "best_done_score": gate, + "max_archive_score": archive.max_archive_score, + "n_cells": len(archive.cells), + "rooms_found": len(archive.rooms), + "explog_entries": explog.count, + "batch": batch, + "nan_flag": 0, + }) + logger.checkpoint(frames, _state_fn, + gate=gate if gate > 0 else None) + + best = archive.best_done_score + final_score = best if best != float("-inf") else 0.0 + hours = (time.time() - t_start) / 3600 + print(f"done: frames {frames:,} cells {len(archive.cells)} rooms {len(archive.rooms)} " + f"best_done {final_score:.0f} ({hours:.2f}h)") + # final.json value_mean = best end-of-episode score (the official Phase-1 + # metric; see targets.yaml montezuma_goexplore for the protocol caveat) + logger.finalize(frames, [final_score], _state_fn, k=1) diff --git a/4-atari-hard/3-robustify.py b/4-atari-hard/3-robustify.py new file mode 100644 index 00000000..30c5347b --- /dev/null +++ b/4-atari-hard/3-robustify.py @@ -0,0 +1,372 @@ +"""Go-Explore robustification (backward algorithm) for Montezuma's Revenge. + +Salimans & Chen 2018 (arXiv:1812.03381) + Go-Explore Nature robustification. +Distills the single deterministic demo found by Phase 1 (2-go-explore.py, score +31,000) into a recurrent policy that plays under STICKY actions — turning a +trajectory-search result into an actual RL policy comparable to RND. + +Mechanism (see env_robustify.py for the curriculum/wrapper details): + episodes restore to a point along the demo and play forward; when the agent + matches the demo's return-to-go from there in >= move_threshold of rollouts, + the starting point marches backward. `max_starting_point -> 0` = the policy + now plays the whole game from reset. That fraction reached is the real + progress metric; the headline number is a from-reset sticky-action eval. + +Single-machine port honest simplifications (vs 128-GPU atari-reset): + - one demo (paper: 40% convergence with one demo even at scale — see SPEC), + - GRU truncated-BPTT over the whole rollout (no separate context window), + - SIL / multi-demo / reward autoscale implemented as OFF-by-default flags. + +Run contract: --seed/--total-frames/--run-dir/--ckpt-every/--resume, plus +--demo (path from extract_demo.py) and --n-envs. +""" +import argparse +import os +import pickle +import time + +import numpy as np +import torch +import torch.nn as nn + +from env_robustify import ReplayResetEnv, ResetManager, load_demo + +try: + from env_go_explore import RunLogger, pick_device # reuse plumbing if present +except Exception: # pick_device may live elsewhere; fall back + from env_go_explore import RunLogger + def pick_device(arg="auto"): + if arg != "auto": + return torch.device(arg) + if torch.cuda.is_available(): + return torch.device("cuda") + if torch.backends.mps.is_available(): + return torch.device("mps") + return torch.device("cpu") + + +TOTAL_FRAMES = 20_000_000 # agent steps (override --total-frames) +N_ENVS = 16 +ROLLOUT = 128 +EPOCHS = 4 +MINIBATCHES = 4 +GAMMA = 0.999 +LAM = 0.95 +CLIP = 0.1 +LR = 1e-4 +ENT_COEF = 1e-5 +VF_COEF = 0.5 +MAX_GRAD_NORM = 0.5 +GRU_DIM = 256 # atari-reset uses 800; 256 keeps MPS light +STICKY = 0.25 +MOVE_THRESHOLD = 0.1 +ADAM_EPS = 1e-6 +LOG_EVERY = 1 # in updates + + +def _ortho(layer, gain): + nn.init.orthogonal_(layer.weight, gain) + nn.init.zeros_(layer.bias) + return layer + + +class GRUActorCritic(nn.Module): + """conv 8/4/3 -> fc + LayerNorm -> GRUCell -> pi, V. Input = 4-stacked + 105x80 grayscale frames (4 channels).""" + + def __init__(self, n_actions, gru_dim=GRU_DIM): + super().__init__() + self.conv = nn.Sequential( + _ortho(nn.Conv2d(4, 32, 8, stride=4), 2 ** 0.5), nn.ReLU(), + _ortho(nn.Conv2d(32, 64, 4, stride=2), 2 ** 0.5), nn.ReLU(), + _ortho(nn.Conv2d(64, 64, 3, stride=1), 2 ** 0.5), nn.ReLU(), + nn.Flatten(), + ) + with torch.no_grad(): + n_flat = self.conv(torch.zeros(1, 4, 105, 80)).shape[1] + self.fc = _ortho(nn.Linear(n_flat, gru_dim), 2 ** 0.5) + self.ln = nn.LayerNorm(gru_dim) + self.gru = nn.GRUCell(gru_dim, gru_dim) + self.pi = _ortho(nn.Linear(gru_dim, n_actions), 0.01) + self.v = _ortho(nn.Linear(gru_dim, 1), 1.0) + self.gru_dim = gru_dim + + def features(self, obs): + h = self.ln(torch.relu(self.fc(self.conv(obs / 255.0)))) + return h + + def step(self, obs, hx, inc_entropy=None): + """One timestep for acting. obs (B,4,105,80), hx (B,gru). Returns + logits, value, new hx.""" + hx = self.gru(self.features(obs), hx) + logits = self.pi(hx) + if inc_entropy is not None: + logits = torch.where(inc_entropy.unsqueeze(1), logits / 2.0, logits) + return logits, self.v(hx).squeeze(-1), hx + + def unroll(self, obs_seq, hx0, done_seq): + """Recompute a (T,B) rollout's logits/values with done-masked GRU state + for BPTT. obs_seq (T,B,4,105,80), hx0 (B,gru), done_seq (T,B).""" + T, B = obs_seq.shape[:2] + hx = hx0 + logits_l, val_l = [], [] + for t in range(T): + hx = hx * (1.0 - done_seq[t]).unsqueeze(1) # reset state after a done + hx = self.gru(self.features(obs_seq[t]), hx) + logits_l.append(self.pi(hx)) + val_l.append(self.v(hx).squeeze(-1)) + return torch.stack(logits_l), torch.stack(val_l) + + +def _stack_init(frame): + return np.repeat(frame[None], 4, axis=0) # (4,105,80) + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--demo", required=True) + p.add_argument("--env", default="montezuma_goexplore_robust") + p.add_argument("--seed", type=int, default=0) + p.add_argument("--total-frames", type=int, default=None) + p.add_argument("--n-envs", type=int, default=None) + p.add_argument("--device", default="auto") + p.add_argument("--run-dir", default=None) + p.add_argument("--ckpt-every", type=int, default=None) + p.add_argument("--resume", default=None) + p.add_argument("--eval-episodes", type=int, default=50) + # stretch flags (off by default — see SPEC) + p.add_argument("--sil", action="store_true") + p.add_argument("--autoscale", action="store_true") + p.add_argument("--ent-coef", type=float, default=ENT_COEF, + help="entropy bonus coefficient (default keeps the module constant). " + "Raise to fight a competence plateau where the policy commits before " + "mastering the demo suffix under sticky actions.") + args = p.parse_args() + global TOTAL_FRAMES, N_ENVS + if args.total_frames: + TOTAL_FRAMES = args.total_frames + if args.n_envs: + N_ENVS = args.n_envs + + torch.manual_seed(args.seed) + np.random.seed(args.seed) + device = pick_device(args.device) + demo = load_demo(args.demo) + print(f"device {device} demo {len(demo['actions'])} actions score {demo['score']:.0f} " + f"n_envs {N_ENVS} total_frames {TOTAL_FRAMES:,} ent_coef {args.ent_coef:g}", flush=True) + + envs = [ReplayResetEnv(demo, seed=args.seed * 1000 + i, sticky=STICKY) for i in range(N_ENVS)] + mgr = ResetManager(demo, N_ENVS, move_threshold=MOVE_THRESHOLD) + mgr.assign(envs) + n_actions = envs[0].env.action_space.n + + net = GRUActorCritic(n_actions).to(device) + opt = torch.optim.Adam(net.parameters(), lr=LR, eps=ADAM_EPS) + logger = RunLogger(args.run_dir, args.ckpt_every) + + # reset all envs + stacks = np.stack([_stack_init(e.reset()) for e in envs]) # (N,4,105,80) + hx = torch.zeros(N_ENVS, net.gru_dim, device=device) + ep_start_nr = np.array([e.start_nr for e in envs]) + frames = 0 + update = 0 + + def _state(): + # RNG: global numpy + CPU torch + per-env numpy Generators (sticky / start-point + # sampling streams). Restored on resume so the kill→resume contract is faithful + # for the deterministic streams (preflight S2). MPS policy-sampling has no bit + # determinism (mps-pitfalls) — these cover what is reproducible. + return {"net": net.state_dict(), "opt": opt.state_dict(), "frames": frames, + "update": update, "max_starting_point": mgr.max_starting_point, + "success": mgr.success, "rng": np.random.get_state(), + "torch_rng": torch.get_rng_state(), + "env_rng": [e.rng.bit_generator.state for e in envs]} + + resume_path = logger.resolve_resume(args.resume) if args.run_dir else None + if resume_path: + ck = torch.load(resume_path, map_location=device, weights_only=False) + net.load_state_dict(ck["net"]); opt.load_state_dict(ck["opt"]) + frames, update = ck["frames"], ck["update"] + mgr.max_starting_point = ck["max_starting_point"]; mgr.success = ck["success"] + np.random.set_state(ck["rng"]); mgr.assign(envs) + if ck.get("torch_rng") is not None: + torch.set_rng_state(ck["torch_rng"].cpu() if hasattr(ck["torch_rng"], "cpu") else ck["torch_rng"]) + for e, st in zip(envs, ck.get("env_rng", [])): + e.rng.bit_generator.state = st + print(f"resumed @ frames {frames} max_start {mgr.max_starting_point}", flush=True) + + n_updates = TOTAL_FRAMES // (ROLLOUT * N_ENVS) + success_window = [] # recent as_good_as_demo outcomes for logging + t0 = time.time() + + while frames < TOTAL_FRAMES: + obs_buf = np.zeros((ROLLOUT, N_ENVS, 4, 105, 80), dtype=np.uint8) + act_buf = np.zeros((ROLLOUT, N_ENVS), dtype=np.int64) + logp_buf = np.zeros((ROLLOUT, N_ENVS), dtype=np.float32) + val_buf = np.zeros((ROLLOUT, N_ENVS), dtype=np.float32) + rew_buf = np.zeros((ROLLOUT, N_ENVS), dtype=np.float32) + done_buf = np.zeros((ROLLOUT, N_ENVS), dtype=np.float32) + rrst_buf = np.zeros((ROLLOUT, N_ENVS), dtype=np.float32) # random_reset mask + ent_buf = np.zeros((ROLLOUT, N_ENVS), dtype=bool) + hx0 = hx.detach().clone() + + for t in range(ROLLOUT): + obs_t = torch.as_tensor(stacks, dtype=torch.float32, device=device) + inc = torch.as_tensor([e.action_nr < e.start_nr + e.inc_entropy_threshold + for e in envs], device=device) + with torch.no_grad(): + logits, value, hx = net.step(obs_t, hx, inc) + dist = torch.distributions.Categorical(logits=logits) + action = dist.sample() + logp = dist.log_prob(action) + obs_buf[t] = stacks + act_buf[t] = action.cpu().numpy() + logp_buf[t] = logp.cpu().numpy() + val_buf[t] = value.cpu().numpy() + ent_buf[t] = inc.cpu().numpy() + + for i, e in enumerate(envs): + frame, r, done, info = e.step(int(act_buf[t, i])) + rew_buf[t, i] = r + done_buf[t, i] = float(done) + rrst_buf[t, i] = float(info.get("random_reset", False)) + # roll the 4-stack + stacks[i] = np.concatenate([stacks[i][1:], frame[None]], axis=0) + if done: + success_window.append(float(info.get("as_good_as_demo", False))) + mgr.record(ep_start_nr[i], info.get("as_good_as_demo", False)) + frame0 = e.reset() + stacks[i] = _stack_init(frame0) + ep_start_nr[i] = e.start_nr + hx[i] = 0.0 # reset recurrent state on episode boundary + frames += N_ENVS + + # bootstrap value + with torch.no_grad(): + obs_t = torch.as_tensor(stacks, dtype=torch.float32, device=device) + _, last_val, _ = net.step(obs_t, hx) + last_val = last_val.cpu().numpy() + + # GAE with random_reset boundaries masked (don't bootstrap across the + # artificial success cutoff) + adv = np.zeros((ROLLOUT, N_ENVS), dtype=np.float32) + gae = np.zeros(N_ENVS, dtype=np.float32) + for t in reversed(range(ROLLOUT)): + nextval = last_val if t == ROLLOUT - 1 else val_buf[t + 1] + nonterm = 1.0 - done_buf[t] + delta = rew_buf[t] + GAMMA * nextval * nonterm - val_buf[t] + gae = delta + GAMMA * LAM * nonterm * gae + gae = gae * (1.0 - rrst_buf[t]) # cut advantage chain at random resets + adv[t] = gae + ret = adv + val_buf + + # flatten time-major for minibatching but keep done seq for GRU unroll + obs_seq = torch.as_tensor(obs_buf, dtype=torch.float32, device=device) + done_seq = torch.as_tensor(done_buf, device=device) + act_t = torch.as_tensor(act_buf, device=device) + oldlogp_t = torch.as_tensor(logp_buf, device=device) + adv_t = torch.as_tensor(adv, device=device) + ret_t = torch.as_tensor(ret, device=device) + ent_t = torch.as_tensor(ent_buf, device=device) + valid = (1.0 - torch.as_tensor(rrst_buf, device=device)) # mask success-cutoff steps + + env_idx = np.arange(N_ENVS) + pl = vl = ent_sum = 0.0 + nmb = 0 + for _ in range(EPOCHS): + np.random.shuffle(env_idx) + mb = max(N_ENVS // MINIBATCHES, 1) + for s in range(0, N_ENVS, mb): + cols = env_idx[s:s + mb] + logits, val = net.unroll(obs_seq[:, cols], hx0[cols], done_seq[:, cols]) + logits = torch.where(ent_t[:, cols].unsqueeze(-1), logits / 2.0, logits) + dist = torch.distributions.Categorical(logits=logits) + newlogp = dist.log_prob(act_t[:, cols]) + m = valid[:, cols] + a = adv_t[:, cols] + a = (a - a.mean()) / (a.std() + 1e-8) + ratio = (newlogp - oldlogp_t[:, cols]).exp() + pg = -torch.min(ratio * a, torch.clamp(ratio, 1 - CLIP, 1 + CLIP) * a) + vloss = (val - ret_t[:, cols]) ** 2 + ent = dist.entropy() + msum = m.sum().clamp(min=1.0) + policy_loss = (pg * m).sum() / msum + value_loss = (vloss * m).sum() / msum + entropy = (ent * m).sum() / msum + loss = policy_loss + VF_COEF * value_loss - args.ent_coef * entropy + opt.zero_grad(); loss.backward() + nn.utils.clip_grad_norm_(net.parameters(), MAX_GRAD_NORM) + opt.step() + pl += policy_loss.item(); vl += value_loss.item() + ent_sum += entropy.item(); nmb += 1 + + mgr.update(envs) + update += 1 + if update % LOG_EVERY == 0: + sw = success_window[-200:] + sps = frames / max(time.time() - t0, 1e-9) + agood = float(np.mean(sw)) if sw else 0.0 + progress = 1.0 - mgr.max_starting_point / max(mgr.max_max, 1) + print(f"upd {update:>5} frames {frames:>9,} max_start {mgr.max_starting_point:>6} " + f"({progress*100:4.1f}% back) as_good {agood:.2f} sps {sps:.0f} " + f"pl {pl/nmb:+.3f} vl {vl/nmb:.3f}", flush=True) + if args.run_dir: + logger.log(frames, { + "max_starting_point": int(mgr.max_starting_point), + "curriculum_progress": progress, + "as_good_as_demo_rate": agood, + "policy_loss": pl / nmb, "value_loss": vl / nmb, + "entropy": ent_sum / nmb, "sps": round(sps, 1), + "game_return_mean_lastK": progress, # gate proxy until eval + "nan_flag": int(not np.isfinite(pl + vl)), + }) + if args.run_dir: + logger.checkpoint(frames, _state, gate=progress) + + # final from-reset sticky eval + score = evaluate(net, demo, device, args.eval_episodes, n_actions, args.seed) + print(f"eval (from reset, sticky): mean {np.mean(score):.0f} n {len(score)}", flush=True) + if args.run_dir: + logger.finalize(frames, score, _state, k=len(score)) + for e in envs: + e.env.close() + + +def evaluate(net, demo, device, n_episodes, n_actions, seed): + """From-reset, sticky-action, eps-greedy 0.0 eval — the RL-policy number.""" + e = ReplayResetEnv(demo, seed=seed + 99, sticky=STICKY, noop_max=30) + e.starting_point = 0 # always from reset + e.frac_sample = 0.0 + # Eval must honor targets montezuma_goexplore_robust.protocol.termination=game_over: + # turn OFF the training-curriculum kills so a from-reset episode runs to a real + # game_over, not a ~allowed_lag-step lag-kill window. Otherwise value_mean reports a + # key-but-slower-than-demo policy as ~0 and the canary's first-key/retreat signal is + # destroyed (preflight S1). lag-kill needs t>allowed_lag & t allowed_lag=n makes + # it unreachable; success-kill needs score>=total_return-deficit -> huge -deficit off. + e.allowed_lag = e.n + e.allowed_score_deficit = -1e18 + e.max_steps = 4500 # standard Montezuma eval cap = 18000 frames / frameskip 4 + # (atari-ale-protocol; same as RND montezuma episode_cap). Bounds + # a passive policy that would otherwise never reach game_over. + scores = [] + for _ in range(n_episodes): + frame = e.reset() + stack = _stack_init(frame) + hx = torch.zeros(1, net.gru_dim, device=device) + ret = 0.0 + done = False + while not done: + with torch.no_grad(): + obs = torch.as_tensor(stack[None], dtype=torch.float32, device=device) + logits, _, hx = net.step(obs, hx) + a = int(logits.argmax(-1)) + frame, _, done, info = e.step(a) + ret += info["raw_reward"] + stack = np.concatenate([stack[1:], frame[None]], axis=0) + scores.append(ret) + e.env.close() + return scores + + +if __name__ == "__main__": + main() diff --git a/4-atari-hard/env.py b/4-atari-hard/env.py new file mode 100644 index 00000000..b44430e3 --- /dev/null +++ b/4-atari-hard/env.py @@ -0,0 +1,240 @@ +"""Atari hard-exploration env setup. + +Two backends: + +* `make_vec_env` returns an **envpool** vector env (C++, multithreaded). RND + on Montezuma needs many parallel envs for the first key to be found by + chance; SyncVectorEnv at 8-16 envs plateaus the policy in the first room + forever. envpool gets us 64-128 envs at ~15k env-steps/s on an M3 with + ~1.3 GB overhead, vs ~1.5k step/s and ballooning memory in Sync. + +* `make_env` is still the gymnasium pipeline so `--test` can pop a window + for human-mode rendering (envpool has no render mode). + +Same preprocessing on both: frameskip 4, 84x84 grayscale, stack 4, sticky +actions (`repeat_action_probability=0.25`), full game-length episodes +(life loss does NOT terminate the episode — intrinsic returns need to +chain across deaths). + +Default game is Montezuma's Revenge. +""" +import argparse +import json +import os +import random +import statistics +import sys +import time +import types + +# envpool eagerly imports a procgen submodule that links against homebrew +# Qt5 on macOS. We don't use procgen — stub both modules before import so +# the rest of envpool loads cleanly on arm64 Macs without brew install qt@5. +sys.modules["envpool.procgen.procgen_envpool"] = types.ModuleType("stub") +sys.modules["envpool.procgen.registration"] = types.ModuleType("stub") + +import ale_py # noqa: E402 (kept for the --test single-env path) +import envpool # noqa: E402 +import gymnasium as gym # noqa: E402 +import numpy as np # noqa: E402 +import pygame # noqa: E402 +import torch # noqa: E402 + +gym.register_envs(ale_py) + + +# --------------------------------------------------------------------------- +# Training utilities (reproducibility, structured logging, resumable runs). +# Shared by the training scripts in this folder. Plain training hygiene — kept +# here next to the other shared plumbing (make_env / parse_args / run_test_loop). +# --------------------------------------------------------------------------- + +def seed_all(seed): + """Seed python / numpy / torch for reproducibility (MPS is not bit-exact).""" + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + + +def _atomic_save(state, path): + """tmp -> rename so a crash mid-write never corrupts the checkpoint.""" + os.makedirs(os.path.dirname(path), exist_ok=True) + tmp = f"{path}.tmp" + torch.save(state, tmp) + os.replace(tmp, path) + + +class RunLogger: + """Optional run-directory outputs: metrics.jsonl, periodic / 5M-milestone / + best checkpoints, resume, and a final.json summary. Inert when run_dir is + None, so the training script still runs standalone.""" + + def __init__(self, run_dir, ckpt_every): + self.dir = run_dir + self.ckpt_dir = os.path.join(run_dir, "ckpt") if run_dir else None + self.ckpt_every = ckpt_every + if self.ckpt_dir: + os.makedirs(self.ckpt_dir, exist_ok=True) + self.f = open(os.path.join(run_dir, "metrics.jsonl"), "a", buffering=1) if run_dir else None + self.t0, self.last_frames = time.time(), 0 + self.ckpt_last, self.ms_last, self.best = 0, 0, float("-inf") + + def log(self, frames, scalars): + """Append one structured row (frames + sps + caller's scalars) to metrics.jsonl.""" + if not self.f: + return + now = time.time() + sps = (frames - self.last_frames) / max(now - self.t0, 1e-9) + self.f.write(json.dumps({"ts": round(now, 1), "frames": frames, "sps": round(sps, 1), **scalars}) + "\n") + self.t0, self.last_frames = now, frames + + def resolve_resume(self, resume_arg): + """'auto' -> run_dir/ckpt/latest.pt, else a path, else None.""" + if resume_arg == "auto" and self.ckpt_dir: + cand = os.path.join(self.ckpt_dir, "latest.pt") + return cand if os.path.exists(cand) else None + if resume_arg and resume_arg != "auto": + return resume_arg if os.path.exists(resume_arg) else None + return None + + def checkpoint(self, frames, state_fn, gate=None): + """Periodic 'latest', 5M-step milestone, and best-gate checkpoints. + state_fn() builds the dict only when a save actually happens.""" + if not self.ckpt_dir or not self.ckpt_every: + return + if frames - self.ckpt_last >= self.ckpt_every: + _atomic_save(state_fn(), os.path.join(self.ckpt_dir, "latest.pt")) + self.ckpt_last = frames + if frames - self.ms_last >= 5_000_000: + _atomic_save(state_fn(), os.path.join(self.ckpt_dir, f"step_{frames // 1_000_000}M.pt")) + self.ms_last = frames + if gate is not None and gate > self.best: + self.best = gate + _atomic_save(state_fn(), os.path.join(self.ckpt_dir, "best.pt")) + + def finalize(self, frames, game_returns, state_fn, k=100): + """Final 'latest' checkpoint + a final.json result summary.""" + if self.ckpt_dir: + _atomic_save(state_fn(), os.path.join(self.ckpt_dir, "latest.pt")) + if self.dir: + tail = [float(x) for x in game_returns[-k:]] + with open(os.path.join(self.dir, "final.json"), "w") as fh: + json.dump({"frames_total": frames, "frames_unit": "agent_steps", + "gate_metric": "game_return_mean_lastK", "K": k, + "value_mean": statistics.fmean(tail) if tail else float("nan"), + "value_std": statistics.pstdev(tail) if len(tail) > 1 else 0.0, + "episodes_counted": len(tail)}, fh, indent=1) + if self.f: + self.f.close() + + +# Gymnasium / ALE id (used by make_env / --test rendering) paired with the +# envpool task name (used by make_vec_env). envpool uses short names without +# the "ALE/" namespace. +ENV_IDS = { + "montezuma": ("ALE/MontezumaRevenge-v5", "MontezumaRevenge-v5"), + "pitfall": ("ALE/Pitfall-v5", "Pitfall-v5"), + "private_eye": ("ALE/PrivateEye-v5", "PrivateEye-v5"), +} + + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument("--env", choices=list(ENV_IDS), default="montezuma", + help="which hard-exploration Atari game to train on") + p.add_argument("--render", action="store_true", + help="open a window during training (single-env --test only)") + p.add_argument("--test", action="store_true", + help="load the saved checkpoint and just play (no learning)") + p.add_argument("--device", choices=["auto", "cpu", "cuda", "mps"], default="auto", + help="override the auto-selected torch device") + p.add_argument("--wandb", action="store_true", + help="log metrics to Weights & Biases") + # --- reproducibility / run-management flags (all optional; omit them and the script runs as before) --- + p.add_argument("--seed", type=int, default=None, + help="reproducibility seed (np/torch/envpool)") + p.add_argument("--total-frames", type=int, default=None, + help="override the in-file TOTAL_FRAMES budget (agent steps)") + p.add_argument("--n-envs", type=int, default=None, + help="override the in-file N_ENVS (e.g. smaller for a smoke run)") + p.add_argument("--run-dir", type=str, default=None, + help="run directory: write metrics.jsonl / ckpt / final.json here") + p.add_argument("--ckpt-every", type=int, default=None, + help="periodic checkpoint interval in agent steps (resume-safe)") + p.add_argument("--resume", type=str, default=None, + help="'auto' (run-dir/ckpt/latest.pt) or a checkpoint path") + return p.parse_args() + + +def make_env(args): + """Single gymnasium env with the standard Atari preprocessing. + + Used for `--test` rendering; envpool has no human render mode.""" + gym_id, _ = ENV_IDS[args.env] + env = gym.make(gym_id, frameskip=1, + render_mode="human" if (args.render or args.test) else None) + env = gym.wrappers.AtariPreprocessing( + env, noop_max=30, frame_skip=4, screen_size=84, + terminal_on_life_loss=False, grayscale_obs=True, scale_obs=False, + ) + env = gym.wrappers.FrameStackObservation(env, stack_size=4) + return env + + +def make_vec_env(args, n_envs, seed=0): + """envpool vector env. Returns (n_envs, 4, 84, 84) uint8 obs and accepts + int32 actions of shape (n_envs,). `info` is a single dict of per-env + arrays; `info["terminated"]` is the real game-over signal (lives==0). + + envpool's `observation_space` / `action_space` are already the single-env + spaces (no `single_*` aliases like gymnasium vector envs).""" + _, pool_id = ENV_IDS[args.env] + return envpool.make_gymnasium( + pool_id, + num_envs=n_envs, + seed=seed, + stack_num=4, + frame_skip=4, + gray_scale=True, + img_height=84, img_width=84, + noop_max=30, + episodic_life=False, # life loss does not end the episode + use_fire_reset=True, # auto-FIRE on reset for games that need it + repeat_action_probability=0.25, # v5-equivalent sticky actions + reward_clip=False, # we sign-clip in the training loop + max_episode_steps=27_000, # standard Atari time limit + ) + + +def pick_device(arg="auto"): + if arg != "auto": + return torch.device(arg) + if torch.cuda.is_available(): + return torch.device("cuda") + if torch.backends.mps.is_available(): + return torch.device("mps") + return torch.device("cpu") + + +def quit_if_window_closed(env): + if not pygame.display.get_init(): + return + for event in pygame.event.get(): + if event.type == pygame.QUIT: + env.close() + sys.exit() + + +def run_test_loop(env, get_action): + """Replay episodes forever using the supplied action picker (single env).""" + while True: + obs, _ = env.reset() + done = False + score = 0.0 + while not done: + quit_if_window_closed(env) + action = get_action(np.asarray(obs)) + obs, reward, terminated, truncated, _ = env.step(action) + done = terminated or truncated + score += reward + print(f"test score: {score}") diff --git a/4-atari-hard/env_go_explore.py b/4-atari-hard/env_go_explore.py new file mode 100644 index 00000000..1e5420b0 --- /dev/null +++ b/4-atari-hard/env_go_explore.py @@ -0,0 +1,152 @@ +"""Go-Explore env setup (restore-based exploration on raw gymnasium ALE). + +Separate plumbing from this folder's `env.py` (the PPO/RND envpool stack): +Go-Explore's exploration phase needs the emulator's save/restore API +(ale.cloneState / restoreState), which envpool does not expose. Each +(worker) process owns a single raw ALE env built by `make_restore_env`. +The harness binds promotion markers to the script PLUS the sibling modules +it actually imports, so 2-go-explore.py is hashed with THIS file, not env.py. + +Protocol (Ecoffet et al. 2019/2021, exploration phase): fully deterministic — +frameskip 4, NO sticky actions, no no-ops, seed 0. Stochasticity only enters +in the (separate, later) robustification phase. The TimeLimit wrapper is +stripped (`.unwrapped`): its step counter is meaningless when episodes are +entered mid-trajectory via state restore. + +★ Verified ALE pitfall (this machine, ale-py 0.11.2): right after +`restoreState`, `getRAM()` / screen reads still return the PRE-restore values +until the next `act()`. Callers must therefore derive cell keys only from +frames returned by `env.step()`, never from immediate post-restore reads. +""" +import argparse +import json +import os +import statistics +import time + +import torch # checkpoint serialization only — there is no neural net here + + +def _atomic_save(state, path): + """tmp -> rename so a crash mid-write never corrupts the checkpoint.""" + os.makedirs(os.path.dirname(path), exist_ok=True) + tmp = f"{path}.tmp" + torch.save(state, tmp) + os.replace(tmp, path) + + +class RunLogger: + """Optional run-directory outputs: metrics.jsonl, periodic / milestone / + best checkpoints, resume, and a final.json summary. Inert when run_dir is + None, so the script still runs standalone. + + Same contract as 4-atari-hard/env.py with one change: milestone + checkpoints fire every 50M frames instead of 5M — a Go-Explore checkpoint + carries the whole archive (~0.5 GB at 50k cells), and a 500M-step run + would otherwise pile up 100 of them.""" + + MILESTONE_EVERY = 50_000_000 + + def __init__(self, run_dir, ckpt_every): + self.dir = run_dir + self.ckpt_dir = os.path.join(run_dir, "ckpt") if run_dir else None + self.ckpt_every = ckpt_every + if self.ckpt_dir: + os.makedirs(self.ckpt_dir, exist_ok=True) + self.f = open(os.path.join(run_dir, "metrics.jsonl"), "a", buffering=1) if run_dir else None + self.t0, self.last_frames = time.time(), 0 + self.ckpt_last, self.ms_last, self.best = 0, 0, float("-inf") + + def log(self, frames, scalars): + """Append one structured row (frames + sps + caller's scalars) to metrics.jsonl.""" + if not self.f: + return + now = time.time() + sps = (frames - self.last_frames) / max(now - self.t0, 1e-9) + self.f.write(json.dumps({"ts": round(now, 1), "frames": frames, "sps": round(sps, 1), **scalars}) + "\n") + self.t0, self.last_frames = now, frames + + def resolve_resume(self, resume_arg): + """'auto' -> run_dir/ckpt/latest.pt, else a path, else None.""" + if resume_arg == "auto" and self.ckpt_dir: + cand = os.path.join(self.ckpt_dir, "latest.pt") + return cand if os.path.exists(cand) else None + if resume_arg and resume_arg != "auto": + return resume_arg if os.path.exists(resume_arg) else None + return None + + def checkpoint(self, frames, state_fn, gate=None): + """Periodic 'latest', 50M-step milestone, and best-gate checkpoints. + state_fn() builds the dict only when a save actually happens.""" + if not self.ckpt_dir or not self.ckpt_every: + return + if frames - self.ckpt_last >= self.ckpt_every: + _atomic_save(state_fn(), os.path.join(self.ckpt_dir, "latest.pt")) + self.ckpt_last = frames + if frames - self.ms_last >= self.MILESTONE_EVERY: + _atomic_save(state_fn(), os.path.join(self.ckpt_dir, f"step_{frames // 1_000_000}M.pt")) + self.ms_last = frames + if gate is not None and gate > self.best: + self.best = gate + _atomic_save(state_fn(), os.path.join(self.ckpt_dir, "best.pt")) + + def finalize(self, frames, game_returns, state_fn, k=100): + """Final 'latest' checkpoint + a final.json result summary.""" + if self.ckpt_dir: + _atomic_save(state_fn(), os.path.join(self.ckpt_dir, "latest.pt")) + if self.dir: + tail = [float(x) for x in game_returns[-k:]] + with open(os.path.join(self.dir, "final.json"), "w") as fh: + json.dump({"frames_total": frames, "frames_unit": "agent_steps", + "gate_metric": "game_return_mean_lastK", "K": k, + "value_mean": statistics.fmean(tail) if tail else float("nan"), + "value_std": statistics.pstdev(tail) if len(tail) > 1 else 0.0, + "episodes_counted": len(tail)}, fh, indent=1) + if self.f: + self.f.close() + + +# Gymnasium / ALE ids. The "_goexplore" key marks a distinct benchmark +# protocol (deterministic, no sticky) — never cross-compare with the +# sticky-action `montezuma` numbers elsewhere in this repo. +ENV_IDS = { + "montezuma_goexplore": "ALE/MontezumaRevenge-v5", +} + + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument("--env", choices=list(ENV_IDS), default="montezuma_goexplore", + help="which game to explore") + # --- reproducibility / run-management flags (harness run contract) --- + p.add_argument("--seed", type=int, default=None, + help="seed for the action RNG (the emulator itself is deterministic)") + p.add_argument("--total-frames", type=int, default=None, + help="override the in-file TOTAL_FRAMES budget (agent steps actually executed)") + p.add_argument("--n-workers", type=int, default=None, + help="override the in-file N_WORKERS (parallel explorer processes)") + p.add_argument("--run-dir", type=str, default=None, + help="run directory: write metrics.jsonl / ckpt / final.json here") + p.add_argument("--ckpt-every", type=int, default=None, + help="periodic checkpoint interval in agent steps (resume-safe)") + p.add_argument("--resume", type=str, default=None, + help="'auto' (run-dir/ckpt/latest.pt) or a checkpoint path") + return p.parse_args() + + +def make_restore_env(env_key): + """Single raw ALE env with clone/restore access. + + Imports live here (not module top) so harness-side tests can stub this + module without pulling in ale_py. Returns the unwrapped env: TimeLimit's + step counter would spuriously truncate restore-based exploration, and + OrderEnforcing rejects step-after-restore patterns.""" + import ale_py + import gymnasium as gym + gym.register_envs(ale_py) + env = gym.make(ENV_IDS[env_key], frameskip=4, + repeat_action_probability=0.0, # deterministic — Phase 1 requirement + obs_type="grayscale").unwrapped + env.reset(seed=0) # canonical deterministic start; variation comes from action RNGs + assert env.spec.kwargs.get("repeat_action_probability", None) == 0.0 + return env diff --git a/4-atari-hard/env_robustify.py b/4-atari-hard/env_robustify.py new file mode 100644 index 00000000..659b4294 --- /dev/null +++ b/4-atari-hard/env_robustify.py @@ -0,0 +1,229 @@ +"""Robustification env plumbing — the backward algorithm of Go-Explore / +Salimans & Chen 2018 (arXiv:1812.03381), distilling a single demo into a +policy that works under sticky actions. + +Faithful-but-small port of openai/atari-reset (+ the uber-research fork the +Nature paper used). Two pieces: + +* `ReplayResetEnv` wraps one raw gymnasium ALE env. Each episode RESTORES to a + point along the demo (`starting_point`) and the agent plays forward from + there under sticky actions. The score counter is seeded with the demo's raw + return up to that point, so "did the agent do as well as the demo from here" + is a single comparison `score >= returns[-1]`. + +* `ResetManager` owns the curriculum: starting points are staggered across the + worker pool near the demo's end and marched BACKWARD as the agent succeeds + (and nudged forward when it collapses). `max_starting_point -> 0` means the + policy now plays the whole game from reset — the real progress metric. + +Design notes (verified against atari-reset wrappers.py / ppo.py): + 1. Success = raw score (incl. demo prefix) >= demo's full return, minus an + allowed deficit. Move rule (code, not paper): new max start = first index + where cumsum(success_rate)/window >= move_threshold; else +nudge forward. + 2. lag kill: stay within `allowed_lag` steps of the demo's pace, compared to + a windowed-min of returns (so faithful play through a negative reward + isn't falsely killed). + 3. success kill: once as-good-as-demo, run exp(U(0,1)*7) extra steps then end + with `random_reset=True` — the trainer masks GAE across this artificial + boundary and the random length stops the agent timing the cutoff. + 4. warm-up replay of the last `reset_steps_ignored` demo actions through the + step path warms the recurrent state; those transitions are `invalid` and + masked from every loss. + 5. trained WITH sticky actions (Go-Explore, not S&C deterministic) so the + policy is robust to the eval protocol; 0-30 no-ops when starting at reset. +""" +import pickle + +import numpy as np + + +class StickyActionEnv: + """repeat_action_probability applied BELOW frameskip — sticky at the raw + action level, the standard v5 stochasticity. We build the ALE env with + sticky 0 and add it here so the demo replay (which must be deterministic) + can bypass it.""" + + def __init__(self, p=0.25): + self.p = p + self.last = 0 + + def reset(self): + self.last = 0 + + def filter(self, action, rng): + if rng.random() < self.p: + return self.last + self.last = action + return action + + +class ReplayResetEnv: + """One raw ALE env that starts episodes from demo states. Not a gym env — + the vectorized loop in 3-robustify.py drives it directly.""" + + def __init__(self, demo, seed, *, sticky=0.25, allowed_lag=50, + allowed_score_deficit=0, reset_steps_ignored=0, + inc_entropy_threshold=100, noop_max=30, max_steps=400_000): + import ale_py + import gymnasium as gym + gym.register_envs(ale_py) + self.env = gym.make(demo["env_id"], frameskip=4, + repeat_action_probability=0.0, # we add sticky ourselves + obs_type="grayscale").unwrapped + self.ale = self.env.ale + self.actions = demo["actions"] + self.rewards = demo["rewards"] + self.returns = demo["returns"] # cumulative raw, return-to-here + self.total_return = float(self.returns[-1]) + self.checkpoints = demo["checkpoints"] + self.ckpt_nr = demo["checkpoint_action_nr"] + self.n = len(self.actions) + self.sticky = StickyActionEnv(sticky) if sticky > 0 else None + self.allowed_lag = allowed_lag + self.allowed_score_deficit = allowed_score_deficit + self.reset_steps_ignored = reset_steps_ignored + self.inc_entropy_threshold = inc_entropy_threshold + self.noop_max = noop_max + self.max_steps = max_steps + self.rng = np.random.default_rng(seed) + self.starting_point = self.n - 1 + self.frac_sample = 0.2 + + # --- frame preprocessing: 105x80 grayscale (atari-reset uses RGB; grayscale + # keeps us light and matches the rest of this repo). 4-stack handled in + # the trainer. Returns uint8 (105, 80). --- + def _frame(self): + import cv2 + g = self.ale.getScreenGrayscale() + return cv2.resize(g, (80, 105), interpolation=cv2.INTER_AREA) + + def _restore_to(self, nr): + """Restore the latest checkpoint at or before nr, replay demo actions + up to nr (no sticky — deterministic), return the post-restore frame + from a real act (never a stale post-restore read).""" + ci = int(np.searchsorted(self.ckpt_nr, nr, side="right") - 1) + ci = max(ci, 0) + self.ale.restoreState(pickle.loads(self.checkpoints[ci])) + replay_from = int(self.ckpt_nr[ci]) + last_frame = None + for i in range(replay_from, nr): + self.ale.act(int(self.actions[i])) + last_frame = None # frames during pure replay are not needed + return last_frame + + def reset(self): + # per-episode starting point: 0.8 at the pinned point, 0.2 uniform tail + if self.rng.random() < self.frac_sample: + nr = int(self.rng.integers(self.starting_point, self.n)) + else: + nr = self.starting_point + if self.sticky: + self.sticky.reset() + + if nr <= 0: + self.env.reset(seed=int(self.rng.integers(2 ** 31))) + for _ in range(int(self.rng.integers(self.noop_max + 1))): + self.ale.act(0) + self.score = 0.0 + self.action_nr = 0 + self.start_nr = 0 + else: + warm = max(nr - self.reset_steps_ignored, 0) + self._restore_to(warm) + self.score = float(self.returns[warm - 1]) if warm > 0 else 0.0 + self.action_nr = warm + self.start_nr = nr # success/entropy measured against the true start + self.extra = 0 + # post-restore screen reads are STALE until the next act — take one real + # NOOP to get a valid frame (not counted toward score/pace). + frame, _ = self._step_raw(0, bookkeep=False) + return frame + + def _step_raw(self, action, *, bookkeep=True): + a = self.sticky.filter(action, self.rng) if self.sticky else action + r = self.ale.act(int(a)) + if bookkeep: + self.score += float(r) + self.action_nr += 1 + return self._frame(), float(r) + + def step(self, action): + frame, raw_r = self._step_raw(action) + info = {"raw_reward": raw_r} + done = False + + # success: as good as the demo from here + if self.extra == 0 and self.score >= self.total_return - self.allowed_score_deficit: + self.extra = int(np.exp(self.rng.random() * 7)) # 1..1096 + if self.extra > 0: + self.extra -= 1 + if self.extra == 0: + done = True + info["random_reset"] = True + info["as_good_as_demo"] = True + + # lag kill: fell behind the demo's pace (windowed-min, deficit-aware) + t = self.action_nr + if not done and t > self.allowed_lag and t < self.n: + lo = max(t - self.allowed_lag, 0) + hi = min(t + self.allowed_lag, self.n) + threshold = float(self.returns[lo:hi].min()) - self.allowed_score_deficit + if self.score < threshold: + done = True + + if self.ale.game_over() or self.action_nr - self.start_nr >= self.max_steps: + done = True + info["increase_entropy"] = (self.action_nr < self.start_nr + self.inc_entropy_threshold) + return frame, np.sign(raw_r), done, info # clipped reward to the agent + + +class ResetManager: + """Owns the shared curriculum across N envs. The trainer calls assign() once + to stagger starting points, and update() each time a batch of episodes + finishes to march max_starting_point backward.""" + + def __init__(self, demo, n_envs, *, move_threshold=0.1, nudge=100, window=None): + self.n = len(demo["actions"]) + self.n_envs = n_envs + self.move_threshold = move_threshold + self.nudge = nudge + # window = the span of staggered starting points (atari-reset nrstartsteps). + # The move target is move_threshold*window of cumulative success mass. + self.window = window or max(n_envs, 32) + self.max_starting_point = self.n - 1 + self.max_max = self.n - 1 + # latest success-rate per starting-point index + self.success = np.zeros(self.n + 1, dtype=np.float64) + + def assign(self, envs): + """Stagger envs across a window below max_starting_point.""" + per = max(self.window // max(self.n_envs, 1), 1) + for i, e in enumerate(envs): + e.starting_point = max(self.max_starting_point - i * per, 0) + + def record(self, starting_point, success): + # exponential-ish freshening: latest wins (atari-reset keeps last rate) + self.success[min(starting_point, self.n)] = float(success) + + def update(self, envs): + """Move rule (atari-reset ResetManager.proc_infos): forward-cumsum the + per-index success rates from index 0; the new max starting point is the + FIRST index where the cumulative mass reaches move_threshold*window — + i.e. march back as far as the practiced success band supports, no + further. If the mass is never reached (success collapsed), nudge the + curriculum forward (easier) by `nudge`.""" + tail = self.success[: self.max_starting_point + 1] + csum = np.cumsum(tail) # forward: mass accumulated up to each index + hits = np.argwhere(csum >= self.move_threshold * self.window) + if len(hits): + new_max = int(hits[0][0]) # earliest index reaching the mass + self.max_starting_point = max(min(new_max, self.max_starting_point), 0) + else: + self.max_starting_point = min(self.max_starting_point + self.nudge, self.max_max) + self.assign(envs) + return self.max_starting_point + + +def load_demo(path): + with open(path, "rb") as f: + return pickle.load(f) diff --git a/4-atari-hard/extract_demo.py b/4-atari-hard/extract_demo.py new file mode 100644 index 00000000..4d0d3822 --- /dev/null +++ b/4-atari-hard/extract_demo.py @@ -0,0 +1,136 @@ +"""Extract a replayable demo from a Go-Explore Phase 1 run, for robustification. + +The backward algorithm (3-robustify.py) needs, for the best trajectory the +archive found: + - actions[] the full action sequence (raw, frameskip-4 agent steps) + - rewards[] per-step RAW (unclipped) game reward + - checkpoints[] periodic pickled ALE states (restore points) + - checkpoint_action_nr[] the action index each checkpoint sits at + +We get actions by walking the experience-log prev_id chain to the DONE cell +(reusing 2-go-explore's ExperienceLog), then replay them on the exact Phase-1 +protocol (deterministic ALE, frameskip 4, sticky 0, seed 0) to capture rewards +and snapshots. The replay's cumulative score must equal the archived DONE +score — same determinism guarantee record-demo.py already relies on; a mismatch +aborts (a non-replayable demo is useless for robustification). Following the +papers we truncate just after the last reward and keep the shortest demo among +the best (here: the single DONE trajectory). + +Usage: + extract_demo.py --run-dir --out [--ckpt-every 512] +""" +import argparse +import importlib.util +import os +import pickle +import sys +from pathlib import Path + +import numpy as np + + +def _load_ge(): + """Import 2-go-explore.py (ExperienceLog/DONE_KEY) with env stubbed.""" + here = Path(__file__).resolve().parent + import types + stub = types.ModuleType("env_go_explore") + stub.ENV_IDS = {"montezuma_goexplore": "ALE/MontezumaRevenge-v5"} + for name in ("RunLogger", "make_restore_env", "parse_args"): + setattr(stub, name, lambda *a, **k: None) + sys.modules["env_go_explore"] = stub + spec = importlib.util.spec_from_file_location("_ge", here / "2-go-explore.py") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + del sys.modules["env_go_explore"] + return mod + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--run-dir", required=True, help="Go-Explore Phase 1 run dir") + p.add_argument("--out", required=True, help="output demo.pkl path") + p.add_argument("--ckpt-every", type=int, default=512, + help="snapshot an ALE restore point every N actions") + p.add_argument("--max-rewards", type=int, default=0, + help="truncate just after the Kth nonzero reward instead of the last " + "(0 = last, default). Use 1 for a first-key-only easy demo — a much " + "shorter horizon for robustification to bootstrap on.") + args = p.parse_args() + + ge = _load_ge() + run_dir = Path(args.run_dir) + + import torch + ckpt_path = run_dir / "ckpt" / "best.pt" + if not ckpt_path.exists(): + ckpt_path = run_dir / "ckpt" / "latest.pt" + ckpt = torch.load(ckpt_path, map_location="cpu", weights_only=False) + done = ckpt["archive"]["cells"].get(ge.DONE_KEY) + if not done: + sys.exit("[extract] no DONE cell — run has no end-of-episode trajectory") + archived_score = done["score"] + print(f"[extract] DONE score {archived_score:.0f}, traj_len {done['traj_len']:,}", flush=True) + + explog = ge.ExperienceLog(str(run_dir / "explog")) + explog.load_state(ckpt["explog"]) + actions = explog.reconstruct_actions(done["traj_last"]) + assert len(actions) == done["traj_len"], (len(actions), done["traj_len"]) + + # replay on the exact Phase-1 protocol, capturing rewards + periodic states + import ale_py + import gymnasium as gym + gym.register_envs(ale_py) + env = gym.make("ALE/MontezumaRevenge-v5", frameskip=4, + repeat_action_probability=0.0).unwrapped + env.reset(seed=0) + rewards, checkpoints, ckpt_nr = [], [], [] + score = 0.0 + last_reward_idx = -1 + for i, a in enumerate(actions): + if i % args.ckpt_every == 0: + checkpoints.append(pickle.dumps(env.ale.cloneState())) + ckpt_nr.append(i) + _, r, term, trunc, _ = env.step(int(a)) + rewards.append(float(r)) + score += float(r) + if r != 0: + last_reward_idx = i + if term or trunc: + break + + if score != archived_score: + sys.exit(f"[extract] REPLAY MISMATCH {score} != {archived_score} — " + f"demo is not replayable, refusing to write") + + # truncate just after a reward (papers: start right before a reward; nothing + # after it helps robustification). --max-rewards K cuts after the Kth reward + # for a shorter, easier-to-bootstrap demo; default cuts after the last. + reward_idxs = [i for i, r in enumerate(rewards) if r != 0.0] + if args.max_rewards > 0 and len(reward_idxs) >= args.max_rewards: + last_reward_idx = reward_idxs[args.max_rewards - 1] + cut = last_reward_idx + 1 + actions, rewards = actions[:cut], rewards[:cut] + checkpoints = [c for c, n in zip(checkpoints, ckpt_nr) if n < cut] + ckpt_nr = [n for n in ckpt_nr if n < cut] + + demo = { + "actions": np.array(actions, dtype=np.int64), + "rewards": np.array(rewards, dtype=np.float32), + "checkpoints": checkpoints, + "checkpoint_action_nr": np.array(ckpt_nr, dtype=np.int64), + "score": float(sum(rewards)), + "returns": np.cumsum(rewards).astype(np.float32), # return-to-here, raw + "env_id": "ALE/MontezumaRevenge-v5", + "protocol": {"frameskip": 4, "sticky": 0.0, "seed": 0}, + "source_run": str(run_dir), + "ale_py": ale_py.__version__, + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, "wb") as f: + pickle.dump(demo, f) + print(f"[extract] wrote {args.out}: {len(actions):,} actions, score {demo['score']:.0f}, " + f"{len(checkpoints)} checkpoints, last reward @ {last_reward_idx}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/4-gym/1-mountaincar/mountaincar_dqn.py b/4-gym/1-mountaincar/mountaincar_dqn.py deleted file mode 100644 index 932ba6b0..00000000 --- a/4-gym/1-mountaincar/mountaincar_dqn.py +++ /dev/null @@ -1,175 +0,0 @@ -import gym -import pylab -import random -import numpy as np -from collections import deque -from keras.layers import Dense -from keras.optimizers import Adam -from keras.models import Sequential - -EPISODES = 4000 - - -class DQNAgent: - def __init__(self, state_size, action_size): - # Cartpole이 학습하는 것을 보려면 "True"로 바꿀 것 - self.render = True - - # state와 action의 크기를 가져와서 모델을 생성하는데 사용함 - self.state_size = state_size - self.action_size = action_size - - # Cartpole DQN 학습의 Hyper parameter 들 - # deque를 통해서 replay memory 생성 - self.discount_factor = 0.99 - self.learning_rate = 0.001 - self.epsilon = 1.0 - self.epsilon_min = 0.005 - self.epsilon_decay = (self.epsilon - self.epsilon_min) / 50000 - self.batch_size = 64 - self.train_start = 1000 - self.memory = deque(maxlen=10000) - - # 학습할 모델과 타겟 모델을 생성 - self.model = self.build_model() - self.target_model = self.build_model() - # 학습할 모델을 타겟 모델로 복사 --> 타겟 모델의 초기화(weight를 같게 해주고 시작해야 함) - self.update_target_model() - - # Deep Neural Network를 통해서 Q Function을 근사 - # state가 입력, 각 행동에 대한 Q Value가 출력인 모델을 생성 - def build_model(self): - model = Sequential() - model.add(Dense(32, input_dim=self.state_size, activation='relu', kernel_initializer='he_uniform')) - model.add(Dense(16, activation='relu', kernel_initializer='he_uniform')) - model.add(Dense(self.action_size, activation='linear', kernel_initializer='he_uniform')) - model.summary() - model.compile(loss='mse', optimizer=Adam(lr=self.learning_rate)) - return model - - # 일정한 시간 간격마다 타겟 모델을 현재 학습하고 있는 모델로 업데이트 - def update_target_model(self): - self.target_model.set_weights(self.model.get_weights()) - - # 행동의 선택은 현재 네트워크에 대해서 epsilon-greedy 정책을 사용 - def get_action(self, state): - if np.random.rand() <= self.epsilon: - return random.randrange(self.action_size) - else: - q_value = self.model.predict(state) - return np.argmax(q_value[0]) - - # 을 replay_memory에 저장함 - def replay_memory(self, state, action, reward, next_state, done): - if action == 2: - action = 1 - self.memory.append((state, action, reward, next_state, done)) - if self.epsilon > self.epsilon_min: - self.epsilon -= self.epsilon_decay - # print(len(self.memory)) - - # replay memory에서 batch_size 만큼의 샘플들을 무작위로 뽑아서 학습 - def train_replay(self): - if len(self.memory) < self.train_start: - return - batch_size = min(self.batch_size, len(self.memory)) - mini_batch = random.sample(self.memory, batch_size) - - update_input = np.zeros((batch_size, self.state_size)) - update_target = np.zeros((batch_size, self.action_size)) - - for i in range(batch_size): - state, action, reward, next_state, done = mini_batch[i] - target = self.model.predict(state)[0] - - # 큐러닝에서와 같이 s'에서의 최대 Q Value를 가져옴. 단, 타겟 모델에서 가져옴 - if done: - target[action] = reward - else: - target[action] = reward + self.discount_factor * \ - np.amax(self.target_model.predict(next_state)[0]) - update_input[i] = state - update_target[i] = target - - # 학습할 정답인 타겟과 현재 자신의 값의 minibatch를 만들고 그것으로 한 번에 모델 업데이트 - self.model.fit(update_input, update_target, batch_size=batch_size, epochs=1, verbose=0) - - # 저장한 모델을 불러옴 - def load_model(self, name): - self.model.load_weights(name) - - # 학습된 모델을 저장함 - def save_model(self, name): - self.model.save_weights(name) - - -if __name__ == "__main__": - # CartPole-v1의 경우 500 타임스텝까지 플레이가능 - env = gym.make('MountainCar-v0') - # 환경으로부터 상태와 행동의 크기를 가져옴 - state_size = env.observation_space.shape[0] - #action_size = env.action_space.n - action_size = 2 - # DQN 에이전트의 생성 - agent = DQNAgent(state_size, action_size) - agent.load_model("./save_model/MountainCar_DQN.h5") - scores, episodes = [], [] - - for e in range(EPISODES): - done = False - score = 0 - state = env.reset() - state = np.reshape(state, [1, state_size]) - print(state) - - # 액션 0(좌), 1(아무것도 안함), 3(아무것도 하지 않는 액션을 하지 않기 위한 fake_action 선언 - fake_action = 0 - - # 같은 액션을 4번하기 위한 카운터 - action_count = 0 - - while not done: - if agent.render: - env.render() - - # 현재 상태에서 행동을 선택하고 한 스텝을 진행 - action_count = action_count + 1 - - if action_count == 4: - action = agent.get_action(state) - action_count = 0 - - if action == 0: - fake_action = 0 - elif action == 1: - fake_action = 2 - - # 선택한 액션으로 1 step을 시행한다 - next_state, reward, done, info = env.step(fake_action) - next_state = np.reshape(next_state, [1, state_size]) - # 에피소드를 끝나게 한 행동에 대해서 -100의 패널티를 줌 - #reward = reward if not done else -100 - - # 을 replay memory에 저장 - agent.replay_memory(state, fake_action, reward, next_state, done) - # 매 타임스텝마다 학습을 진행 - agent.train_replay() - score += reward - state = next_state - - if done: - env.reset() - # 매 에피소드마다 학습하는 모델을 타겟 모델로 복사 - agent.update_target_model() - - # 각 에피소드마다 cartpole이 서있었던 타임스텝을 plot - scores.append(score) - episodes.append(e) - #pylab.plot(episodes, scores, 'b') - #pylab.savefig("./save_graph/MountainCar_DQN.png") - print("episode:", e, " score:", score, " memory length:", len(agent.memory), - " epsilon:", agent.epsilon) - - # 50 에피소드마다 학습 모델을 저장 - if e % 50 == 0: - agent.save_model("./save_model/MountainCar_DQN.h5") diff --git a/4-gym/1-mountaincar/save_model/MountainCar_DQN.h5 b/4-gym/1-mountaincar/save_model/MountainCar_DQN.h5 deleted file mode 100644 index 7f17c818..00000000 Binary files a/4-gym/1-mountaincar/save_model/MountainCar_DQN.h5 and /dev/null differ diff --git a/README.md b/README.md index 870e686b..936ef190 100644 --- a/README.md +++ b/README.md @@ -1,55 +1,100 @@

--------------------------------------------------------------------------------- - -> Minimal and clean examples of reinforcement learning algorithms presented by [RLCode](https://rlcode.github.io) team. [[한국어]](https://github.com/rlcode/reinforcement-learning-kr) -> -> Maintainers - [Woongwon](https://github.com/dnddnjs), [Youngmoo](https://github.com/zzing0907), [Hyeokreal](https://github.com/Hyeokreal), [Uiryeong](https://github.com/wooridle), [Keon](https://github.com/keon) - -From the basics to deep reinforcement learning, this repo provides easy-to-read code examples. One file for each algorithm. -Please feel free to create a [Pull Request](https://github.com/rlcode/reinforcement-learning/pulls), or open an [issue](https://github.com/rlcode/reinforcement-learning/issues)! - -## Dependencies -1. Python 3.5 -2. Tensorflow 1.0.0 -3. Keras -4. numpy -5. pandas -6. matplot -7. pillow -8. Skimage -9. h5py - -### Install Requirements -``` -pip install -r requirements.txt +From the basics to deep reinforcement learning, this repo provides easy-to-read code examples. One file for each algorithm. Please feel free to create a Pull Request, or open an issue! + +## Algorithms + +**Grid World** ([`1-grid-world/`](./1-grid-world)) + +1. Policy Iteration — [`1-policy_iteration.py`](./1-grid-world/1-policy_iteration.py) +2. Value Iteration — [`2-value_iteration.py`](./1-grid-world/2-value_iteration.py) +3. SARSA — [`3-sarsa.py`](./1-grid-world/3-sarsa.py) +4. Q-Learning — [`4-q_learning.py`](./1-grid-world/4-q_learning.py) +5. Deep SARSA — [`5-deep_sarsa.py`](./1-grid-world/5-deep_sarsa.py) +6. REINFORCE — [`6-reinforce.py`](./1-grid-world/6-reinforce.py) + +**CartPole** ([`2-cartpole/`](./2-cartpole)) + +7. DQN — [`1-dqn.py`](./2-cartpole/1-dqn.py) +8. A2C — [`2-a2c.py`](./2-cartpole/2-a2c.py) +9. PPO — [`3-ppo.py`](./2-cartpole/3-ppo.py) + +**Atari** ([`3-atari/`](./3-atari)) + +10. DQN — [`1-dqn.py`](./3-atari/1-dqn.py) +11. PPO — [`2-ppo.py`](./3-atari/2-ppo.py) + +## Benchmarks + +Trained on a **MacBook Pro 14" (Apple M3, 8 GB unified memory)**, macOS 26.2, Python 3.11, PyTorch 2.11 with the MPS backend. CPU / GPU figures are read from Activity Monitor on the `python3.11` process after the run has stabilized (~5 min in); peak RAM is the process's real memory at its high-water mark. Final score is the mean per-game return over the last 20 episodes of training. + +### Atari — Breakout (10M agent steps, `ALE/Breakout-v5` with sticky actions) + +| Algorithm | Params | Train time | Final mean (per-game) | Peak RAM | CPU% | GPU% | W&B | +|-----------|--------|------------|-----------------------|----------|------|------|-----| +| DQN | 1.69M | ~9h | 93.5 ± 9.6 | 5.27 GB | ~60 | ~55 | [report](https://api.wandb.ai/links/rlcode/ljkn7ahp) | +| PPO | 1.69M | ~3.8h | 261.9 ± 6.4 | 1.98 GB | ~62 | ~55 | [report](https://api.wandb.ai/links/rlcode/jbdsbn6t) | + +> Single seed per row, mean ± std over the final 20 logged episodes. `Params` counts only trainable network weights. `CPU%` is the single-process value reported by Activity Monitor (sum across cores, so >100% means multi-core use); `GPU%` is the same column for the Apple GPU. Sticky actions (`repeat_action_probability=0.25`) make absolute scores lower than the deterministic `*-v4` environments often cited in older papers. + +### Atari — Montezuma's Revenge + +Mac Studio (Apple M4 Max, 64 GB), `ALE/MontezumaRevenge-v5`, single seed. **Two protocols, not cross-comparable:** sticky-action RL vs deterministic restore-based search. + +| Method | Protocol | Score (single seed) | Frames | Link | +|--------|----------|---------------------|--------|------| +| PPO + RND ([`1-ppo-rnd.py`](./4-atari-hard/1-ppo-rnd.py)) | sticky, RL policy | ~3,120 | 65M | [report](https://api.wandb.ai/links/rlcode/3j0nfk9s) | +| Go-Explore — exploration ([`2-go-explore.py`](./4-atari-hard/2-go-explore.py)) | deterministic search | 31,000 (replay-verified) | 500M | [run](https://wandb.ai/rlcode/rl-atari-hard-go-explore/runs/m6ox4l3m) | +| Go-Explore — robustification ([`3-robustify.py`](./4-atari-hard/3-robustify.py)) | sticky, RL policy | — (no from-reset score) | 5M | — | + +> **RND** (Burda et al. 2018): first key ~327k steps with 512 envs (128 never scored in 50M — parallel breadth is the lever); plateaued above the PPO baseline 2497, below RND's 8152 (~30× more experience). **Exploration** (Ecoffet et al. 2019/2021): best end-of-episode trajectory from a knowledge-free cell archive, no NN — a search result, not an RL score (Nature ref 24,758). **Robustification** (backward algorithm, Salimans & Chen 2018): bootstraps with a first-key demo + 128 envs but the curriculum plateaus ~22% on one machine — no from-reset score, a scale ceiling vs the original hundreds–thousands of envs. + +## Setup + +Requires Python 3.11 and [uv](https://docs.astral.sh/uv/). + +```bash +git clone +cd reinforcement-learning +uv sync ``` -## Table of Contents +## Running + +```bash +# Grid World +cd 1-grid-world && uv run python 3-sarsa.py + +# CartPole — train +cd 2-cartpole && uv run python 1-dqn.py -**Grid World** - Mastering the basics of reinforcement learning in the simplified world called "Grid World" +# CartPole — watch training (slower) +cd 2-cartpole && uv run python 1-dqn.py --render -- [Policy Iteration](./1-grid-world/1-policy-iteration) -- [Value Iteration](./1-grid-world/2-value-iteration) -- [Monte Carlo](./1-grid-world/3-monte-carlo) -- [SARSA](./1-grid-world/4-sarsa) -- [Q-Learning](./1-grid-world/5-q-learning) -- [Deep SARSA](./1-grid-world/6-deep-sarsa) -- [REINFORCE](./1-grid-world/7-reinforce) +# CartPole — replay a trained checkpoint +cd 2-cartpole && uv run python 1-dqn.py --test +``` + +### Logging to Weights & Biases (Atari only) -**CartPole** - Applying deep reinforcement learning on basic Cartpole game. +Both Atari scripts (`1-dqn.py`, `2-ppo.py`) can stream training metrics to your own [Weights & Biases](https://wandb.ai/) account. One-time login, then pass `--wandb`: -- [Deep Q Network](./2-cartpole/1-dqn) -- [Double Deep Q Network](./2-cartpole/2-double-dqn) -- [Policy Gradient](./2-cartpole/3-reinforce) -- [Actor Critic (A2C)](./2-cartpole/4-actor-critic) -- [Asynchronous Advantage Actor Critic (A3C)](./2-cartpole/5-a3c) +```bash +uv run wandb login # paste the API key from https://wandb.ai/authorize +cd 3-atari && uv run python 2-ppo.py --env breakout --wandb +cd 3-atari && uv run python 1-dqn.py --env breakout --wandb +``` -**Atari** - Mastering Atari games with Deep Reinforcement Learning +Runs land in *your* `rl-atari-ppo` / `rl-atari-dqn` project — nothing is shared by default. Omit `--wandb` and the script runs without ever touching the network. -- **Breakout** - [DQN](./3-atari/1-breakout/breakout_dqn.py), [DDQN](./3-atari/1-breakout/breakout_ddqn.py) [Dueling DDQN](./3-atari/1-breakout/breakout_ddqn.py) [A3C](./3-atari/1-breakout/breakout_a3c.py) -- **Pong** - [Policy Gradient](./3-atari/2-pong/pong_reinforce.py) +## Updates -**OpenAI GYM** - [WIP] +Modernized from the 2017 original: -- Mountain Car - [DQN](./4-gym/1-mountaincar) +- **Framework**: Keras + TensorFlow 1.0 → PyTorch 2.11 +- **Env**: gym 0.8 → gymnasium 1.2 +- **Rendering**: tkinter → pygame (cross-platform with no system Tk) +- **Tooling**: `requirements.txt` → `pyproject.toml` + `uv` +- **Scope**: pruned to 9 core algorithms; dropped Monte Carlo / DDQN / A3C / Atari / mountaincar; added PPO +- **Layout**: flat `1-grid-world/3-sarsa.py` instead of nested `1-grid-world/4-sarsa/sarsa_agent.py` +- **Docs**: each algorithm file now opens with a paper citation and the core update equation diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..2426d4e8 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,18 @@ +[project] +name = "reinforcement-learning" +version = "0.1.0" +description = "Minimal PyTorch reimplementation of classic RL algorithms" +requires-python = "==3.11.*" +dependencies = [ + "torch>=2.11,<2.12", + "torchvision>=0.26,<0.27", + "gymnasium[atari]>=1.2,<1.3", + "ale-py>=0.11,<0.12", + "numpy>=2.3,<2.4", + "matplotlib>=3.10,<3.11", + "pygame>=2.6,<3", + "opencv-python-headless>=4.13,<4.14", + "wandb>=0.27.0", + "moviepy>=2.2.1", + "envpool>=1.2.5", +] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index d2b5b0fc..00000000 --- a/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -Keras==2.0.3 -numpy==1.12.1 -pandas==0.19.2 -matplotlib==2.0.0 -tensorflow==1.0.0 -Pillow==4.1.0 -gym==0.8.1 -h5py==2.7.0 -scikit-image==0.13.0 diff --git a/uv.lock b/uv.lock new file mode 100644 index 00000000..dadb86fe --- /dev/null +++ b/uv.lock @@ -0,0 +1,1335 @@ +version = 1 +revision = 3 +requires-python = "==3.11.*" + +[[package]] +name = "absl-py" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/64/c7/8de93764ad66968d19329a7e0c147a2bb3c7054c554d4a119111b8f9440f/absl_py-2.4.0.tar.gz", hash = "sha256:8c6af82722b35cf71e0f4d1d47dcaebfff286e27110a99fc359349b247dfb5d4", size = 116543 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/a6/907a406bb7d359e6a63f99c313846d9eec4f7e6f7437809e03aa00fa3074/absl_py-2.4.0-py3-none-any.whl", hash = "sha256:88476fd881ca8aab94ffa78b7b6c632a782ab3ba1cd19c9bd423abc4fb4cd28d", size = 135750 }, +] + +[[package]] +name = "ale-py" +version = "0.11.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2c/85/fffbe95501efc9ecf73e02fb62e7a99401e12dcb8217313016dd00b13cdc/ale_py-0.11.2.tar.gz", hash = "sha256:cc6fed9d994d796d76b0d042fbe0a7101834faa55b5e70b1e1f14367ed9e2a8a", size = 512258, upload-time = "2025-07-12T22:19:16.348Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/9c/ac9735b7a8776323fba54a0a3a8eac4706edb8fdac8a621516550b77ee14/ale_py-0.11.2-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:808c98685a607cc5483238f73915c23426537259f9cece506f47f5213c370734", size = 2337280, upload-time = "2025-07-12T22:18:48.839Z" }, + { url = "https://files.pythonhosted.org/packages/7b/00/9ec89f60c14a378096f23c190a60155b9551e521a00046dd8bb3279b00ea/ale_py-0.11.2-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:d80311cf92ca6ca777dec363865891dbb5447e0c9f57774f72c8618851c9fd4b", size = 2467834, upload-time = "2025-07-12T22:18:50.072Z" }, + { url = "https://files.pythonhosted.org/packages/13/39/293fab6925966076112a49683ac71ccfe6cd75a1790856e90e3ef0a11bd1/ale_py-0.11.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:44838121ab5c2ef50033ebf0cc69aadf3954418d2f8812bdb76fced3797eb33f", size = 5067629, upload-time = "2025-07-12T22:18:51.554Z" }, + { url = "https://files.pythonhosted.org/packages/3d/92/623e8b3157fdd39a0d78026a1c2727c9215d882d67bfb72697661c7dba6f/ale_py-0.11.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f7a2082e00fc81b6706daf945bd4c97b69c5542739707638c65ddf65ad74db38", size = 5054969, upload-time = "2025-07-12T22:18:53.177Z" }, + { url = "https://files.pythonhosted.org/packages/8f/41/a7de8030021b478e8829b47e8667e79c910c3bb1539d72b6c8052b14cc3c/ale_py-0.11.2-cp311-cp311-win_amd64.whl", hash = "sha256:858a644ed92409cdef47a88d177d18421260b74d3f5cdb45963f21de870a6fd9", size = 3471162, upload-time = "2025-07-12T22:18:54.343Z" }, +] + +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548 }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548 }, +] + +[[package]] +name = "certifi" +version = "2026.4.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/25/ee/6caf7a40c36a1220410afe15a1cc64993a1f864871f698c0f93acb72842a/certifi-2026.4.22.tar.gz", hash = "sha256:8d455352a37b71bf76a79caa83a3d6c25afee4a385d632127b6afb3963f1c580", size = 137077, upload-time = "2026-04-22T11:26:11.191Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl", hash = "sha256:3cb2210c8f88ba2318d29b0388d1023c8492ff72ecdde4ebdaddbb13a31b1c4a", size = 135707, upload-time = "2026-04-22T11:26:09.372Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/a1/67fe25fac3c7642725500a3f6cfe5821ad557c3abb11c9d20d12c7008d3e/charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5", size = 144271, upload-time = "2026-04-02T09:28:39.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/d7/b5b7020a0565c2e9fa8c09f4b5fa6232feb326b8c20081ccded47ea368fd/charset_normalizer-3.4.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7641bb8895e77f921102f72833904dcd9901df5d6d72a2ab8f31d04b7e51e4e7", size = 309705, upload-time = "2026-04-02T09:26:02.191Z" }, + { url = "https://files.pythonhosted.org/packages/5a/53/58c29116c340e5456724ecd2fff4196d236b98f3da97b404bc5e51ac3493/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:202389074300232baeb53ae2569a60901f7efadd4245cf3a3bf0617d60b439d7", size = 206419, upload-time = "2026-04-02T09:26:03.583Z" }, + { url = "https://files.pythonhosted.org/packages/b2/02/e8146dc6591a37a00e5144c63f29fb7c97a734ea8a111190783c0e60ab63/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30b8d1d8c52a48c2c5690e152c169b673487a2a58de1ec7393196753063fcd5e", size = 227901, upload-time = "2026-04-02T09:26:04.738Z" }, + { url = "https://files.pythonhosted.org/packages/fb/73/77486c4cd58f1267bf17db420e930c9afa1b3be3fe8c8b8ebbebc9624359/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:532bc9bf33a68613fd7d65e4b1c71a6a38d7d42604ecf239c77392e9b4e8998c", size = 222742, upload-time = "2026-04-02T09:26:06.36Z" }, + { url = "https://files.pythonhosted.org/packages/a1/fa/f74eb381a7d94ded44739e9d94de18dc5edc9c17fb8c11f0a6890696c0a9/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2fe249cb4651fd12605b7288b24751d8bfd46d35f12a20b1ba33dea122e690df", size = 214061, upload-time = "2026-04-02T09:26:08.347Z" }, + { url = "https://files.pythonhosted.org/packages/dc/92/42bd3cefcf7687253fb86694b45f37b733c97f59af3724f356fa92b8c344/charset_normalizer-3.4.7-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:65bcd23054beab4d166035cabbc868a09c1a49d1efe458fe8e4361215df40265", size = 199239, upload-time = "2026-04-02T09:26:09.823Z" }, + { url = "https://files.pythonhosted.org/packages/4c/3d/069e7184e2aa3b3cddc700e3dd267413dc259854adc3380421c805c6a17d/charset_normalizer-3.4.7-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:08e721811161356f97b4059a9ba7bafb23ea5ee2255402c42881c214e173c6b4", size = 210173, upload-time = "2026-04-02T09:26:10.953Z" }, + { url = "https://files.pythonhosted.org/packages/62/51/9d56feb5f2e7074c46f93e0ebdbe61f0848ee246e2f0d89f8e20b89ebb8f/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e060d01aec0a910bdccb8be71faf34e7799ce36950f8294c8bf612cba65a2c9e", size = 209841, upload-time = "2026-04-02T09:26:12.142Z" }, + { url = "https://files.pythonhosted.org/packages/d2/59/893d8f99cc4c837dda1fe2f1139079703deb9f321aabcb032355de13b6c7/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:38c0109396c4cfc574d502df99742a45c72c08eff0a36158b6f04000043dbf38", size = 200304, upload-time = "2026-04-02T09:26:13.711Z" }, + { url = "https://files.pythonhosted.org/packages/7d/1d/ee6f3be3464247578d1ed5c46de545ccc3d3ff933695395c402c21fa6b77/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:1c2a768fdd44ee4a9339a9b0b130049139b8ce3c01d2ce09f67f5a68048d477c", size = 229455, upload-time = "2026-04-02T09:26:14.941Z" }, + { url = "https://files.pythonhosted.org/packages/54/bb/8fb0a946296ea96a488928bdce8ef99023998c48e4713af533e9bb98ef07/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:1a87ca9d5df6fe460483d9a5bbf2b18f620cbed41b432e2bddb686228282d10b", size = 210036, upload-time = "2026-04-02T09:26:16.478Z" }, + { url = "https://files.pythonhosted.org/packages/9a/bc/015b2387f913749f82afd4fcba07846d05b6d784dd16123cb66860e0237d/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d635aab80466bc95771bb78d5370e74d36d1fe31467b6b29b8b57b2a3cd7d22c", size = 224739, upload-time = "2026-04-02T09:26:17.751Z" }, + { url = "https://files.pythonhosted.org/packages/17/ab/63133691f56baae417493cba6b7c641571a2130eb7bceba6773367ab9ec5/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ae196f021b5e7c78e918242d217db021ed2a6ace2bc6ae94c0fc596221c7f58d", size = 216277, upload-time = "2026-04-02T09:26:18.981Z" }, + { url = "https://files.pythonhosted.org/packages/06/6d/3be70e827977f20db77c12a97e6a9f973631a45b8d186c084527e53e77a4/charset_normalizer-3.4.7-cp311-cp311-win32.whl", hash = "sha256:adb2597b428735679446b46c8badf467b4ca5f5056aae4d51a19f9570301b1ad", size = 147819, upload-time = "2026-04-02T09:26:20.295Z" }, + { url = "https://files.pythonhosted.org/packages/20/d9/5f67790f06b735d7c7637171bbfd89882ad67201891b7275e51116ed8207/charset_normalizer-3.4.7-cp311-cp311-win_amd64.whl", hash = "sha256:8e385e4267ab76874ae30db04c627faaaf0b509e1ccc11a95b3fc3e83f855c00", size = 159281, upload-time = "2026-04-02T09:26:21.74Z" }, + { url = "https://files.pythonhosted.org/packages/ca/83/6413f36c5a34afead88ce6f66684d943d91f233d76dd083798f9602b75ae/charset_normalizer-3.4.7-cp311-cp311-win_arm64.whl", hash = "sha256:d4a48e5b3c2a489fae013b7589308a40146ee081f6f509e047e0e096084ceca1", size = 147843, upload-time = "2026-04-02T09:26:22.901Z" }, + { url = "https://files.pythonhosted.org/packages/db/8f/61959034484a4a7c527811f4721e75d02d653a35afb0b6054474d8185d4c/charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d", size = 61958, upload-time = "2026-04-02T09:28:37.794Z" }, +] + +[[package]] +name = "click" +version = "8.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/23/e4/796662cd90cf80e3a363c99db2b88e0e394b988a575f60a17e16440cd011/click-8.4.0.tar.gz", hash = "sha256:638f1338fe1235c8f4e008e4a8a254fb5c5fbdcbb40ece3c9142ebb78e792973", size = 350843, upload-time = "2026-05-17T00:47:58.425Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ee/ae/8e92f8058baf87f6c7d86ee7e457668690195cc77efedb8d3797a06e3940/click-8.4.0-py3-none-any.whl", hash = "sha256:40c50b7c6c6adac2823d411041ec84f3f103f1b280d5e9ce0d7f998995832f81", size = 116147, upload-time = "2026-05-17T00:47:56.842Z" }, +] + +[[package]] +name = "cloudpickle" +version = "3.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330, upload-time = "2025-11-03T09:25:26.604Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228, upload-time = "2025-11-03T09:25:25.534Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "contourpy" +version = "1.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/58/01/1253e6698a07380cd31a736d248a3f2a50a7c88779a1813da27503cadc2a/contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880", size = 13466174, upload-time = "2025-07-26T12:03:12.549Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/2e/c4390a31919d8a78b90e8ecf87cd4b4c4f05a5b48d05ec17db8e5404c6f4/contourpy-1.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:709a48ef9a690e1343202916450bc48b9e51c049b089c7f79a267b46cffcdaa1", size = 288773, upload-time = "2025-07-26T12:01:02.277Z" }, + { url = "https://files.pythonhosted.org/packages/0d/44/c4b0b6095fef4dc9c420e041799591e3b63e9619e3044f7f4f6c21c0ab24/contourpy-1.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:23416f38bfd74d5d28ab8429cc4d63fa67d5068bd711a85edb1c3fb0c3e2f381", size = 270149, upload-time = "2025-07-26T12:01:04.072Z" }, + { url = "https://files.pythonhosted.org/packages/30/2e/dd4ced42fefac8470661d7cb7e264808425e6c5d56d175291e93890cce09/contourpy-1.3.3-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:929ddf8c4c7f348e4c0a5a3a714b5c8542ffaa8c22954862a46ca1813b667ee7", size = 329222, upload-time = "2025-07-26T12:01:05.688Z" }, + { url = "https://files.pythonhosted.org/packages/f2/74/cc6ec2548e3d276c71389ea4802a774b7aa3558223b7bade3f25787fafc2/contourpy-1.3.3-cp311-cp311-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e999574eddae35f1312c2b4b717b7885d4edd6cb46700e04f7f02db454e67c1", size = 377234, upload-time = "2025-07-26T12:01:07.054Z" }, + { url = "https://files.pythonhosted.org/packages/03/b3/64ef723029f917410f75c09da54254c5f9ea90ef89b143ccadb09df14c15/contourpy-1.3.3-cp311-cp311-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0bf67e0e3f482cb69779dd3061b534eb35ac9b17f163d851e2a547d56dba0a3a", size = 380555, upload-time = "2025-07-26T12:01:08.801Z" }, + { url = "https://files.pythonhosted.org/packages/5f/4b/6157f24ca425b89fe2eb7e7be642375711ab671135be21e6faa100f7448c/contourpy-1.3.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51e79c1f7470158e838808d4a996fa9bac72c498e93d8ebe5119bc1e6becb0db", size = 355238, upload-time = "2025-07-26T12:01:10.319Z" }, + { url = "https://files.pythonhosted.org/packages/98/56/f914f0dd678480708a04cfd2206e7c382533249bc5001eb9f58aa693e200/contourpy-1.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:598c3aaece21c503615fd59c92a3598b428b2f01bfb4b8ca9c4edeecc2438620", size = 1326218, upload-time = "2025-07-26T12:01:12.659Z" }, + { url = "https://files.pythonhosted.org/packages/fb/d7/4a972334a0c971acd5172389671113ae82aa7527073980c38d5868ff1161/contourpy-1.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:322ab1c99b008dad206d406bb61d014cf0174df491ae9d9d0fac6a6fda4f977f", size = 1392867, upload-time = "2025-07-26T12:01:15.533Z" }, + { url = "https://files.pythonhosted.org/packages/75/3e/f2cc6cd56dc8cff46b1a56232eabc6feea52720083ea71ab15523daab796/contourpy-1.3.3-cp311-cp311-win32.whl", hash = "sha256:fd907ae12cd483cd83e414b12941c632a969171bf90fc937d0c9f268a31cafff", size = 183677, upload-time = "2025-07-26T12:01:17.088Z" }, + { url = "https://files.pythonhosted.org/packages/98/4b/9bd370b004b5c9d8045c6c33cf65bae018b27aca550a3f657cdc99acdbd8/contourpy-1.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:3519428f6be58431c56581f1694ba8e50626f2dd550af225f82fb5f5814d2a42", size = 225234, upload-time = "2025-07-26T12:01:18.256Z" }, + { url = "https://files.pythonhosted.org/packages/d9/b6/71771e02c2e004450c12b1120a5f488cad2e4d5b590b1af8bad060360fe4/contourpy-1.3.3-cp311-cp311-win_arm64.whl", hash = "sha256:15ff10bfada4bf92ec8b31c62bf7c1834c244019b4a33095a68000d7075df470", size = 193123, upload-time = "2025-07-26T12:01:19.848Z" }, + { url = "https://files.pythonhosted.org/packages/a5/29/8dcfe16f0107943fa92388c23f6e05cff0ba58058c4c95b00280d4c75a14/contourpy-1.3.3-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:cd5dfcaeb10f7b7f9dc8941717c6c2ade08f587be2226222c12b25f0483ed497", size = 278809, upload-time = "2025-07-26T12:02:52.74Z" }, + { url = "https://files.pythonhosted.org/packages/85/a9/8b37ef4f7dafeb335daee3c8254645ef5725be4d9c6aa70b50ec46ef2f7e/contourpy-1.3.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:0c1fc238306b35f246d61a1d416a627348b5cf0648648a031e14bb8705fcdfe8", size = 261593, upload-time = "2025-07-26T12:02:54.037Z" }, + { url = "https://files.pythonhosted.org/packages/0a/59/ebfb8c677c75605cc27f7122c90313fd2f375ff3c8d19a1694bda74aaa63/contourpy-1.3.3-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70f9aad7de812d6541d29d2bbf8feb22ff7e1c299523db288004e3157ff4674e", size = 302202, upload-time = "2025-07-26T12:02:55.947Z" }, + { url = "https://files.pythonhosted.org/packages/3c/37/21972a15834d90bfbfb009b9d004779bd5a07a0ec0234e5ba8f64d5736f4/contourpy-1.3.3-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5ed3657edf08512fc3fe81b510e35c2012fbd3081d2e26160f27ca28affec989", size = 329207, upload-time = "2025-07-26T12:02:57.468Z" }, + { url = "https://files.pythonhosted.org/packages/0c/58/bd257695f39d05594ca4ad60df5bcb7e32247f9951fd09a9b8edb82d1daa/contourpy-1.3.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:3d1a3799d62d45c18bafd41c5fa05120b96a28079f2393af559b843d1a966a77", size = 225315, upload-time = "2025-07-26T12:02:58.801Z" }, +] + +[[package]] +name = "cuda-bindings" +version = "13.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-pathfinder" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/a9/3a8241c6e19483ac1f1dcf5c10238205dcb8a6e9d0d4d4709240dff28ff4/cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:721104c603f059780d287969be3d194a18d0cc3b713ed9049065a1107706759d", size = 5730273, upload-time = "2026-03-11T00:12:37.18Z" }, + { url = "https://files.pythonhosted.org/packages/e9/94/2748597f47bb1600cd466b20cab4159f1530a3a33fe7f70fee199b3abb9e/cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1eba9504ac70667dd48313395fe05157518fd6371b532790e96fbb31bbb5a5e1", size = 6313924, upload-time = "2026-03-11T00:12:39.462Z" }, +] + +[[package]] +name = "cuda-pathfinder" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/11/d0/c177e29701cf1d3008d7d2b16b5fc626592ce13bd535f8795c5f57187e0e/cuda_pathfinder-1.5.4-py3-none-any.whl", hash = "sha256:9563d3175ce1828531acf4b94e1c1c7d67208c347ca002493e2654878b26f4b7", size = 51657, upload-time = "2026-04-27T22:42:07.712Z" }, +] + +[[package]] +name = "cuda-toolkit" +version = "13.0.2" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/57/b2/453099f5f3b698d7d0eab38916aac44c7f76229f451709e2eb9db6615dcd/cuda_toolkit-13.0.2-py2.py3-none-any.whl", hash = "sha256:b198824cf2f54003f50d64ada3a0f184b42ca0846c1c94192fa269ecd97a66eb", size = 2364, upload-time = "2025-12-19T23:24:07.328Z" }, +] + +[package.optional-dependencies] +cublas = [ + { name = "nvidia-cublas", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +cudart = [ + { name = "nvidia-cuda-runtime", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +cufft = [ + { name = "nvidia-cufft", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +cufile = [ + { name = "nvidia-cufile", marker = "sys_platform == 'linux'" }, +] +cupti = [ + { name = "nvidia-cuda-cupti", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +curand = [ + { name = "nvidia-curand", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +cusolver = [ + { name = "nvidia-cusolver", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +cusparse = [ + { name = "nvidia-cusparse", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +nvjitlink = [ + { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +nvrtc = [ + { name = "nvidia-cuda-nvrtc", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +nvtx = [ + { name = "nvidia-nvtx", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] + +[[package]] +name = "cycler" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a9/95/a3dbbb5028f35eafb79008e7522a75244477d2838f38cbb722248dabc2a8/cycler-0.12.1.tar.gz", hash = "sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c", size = 7615, upload-time = "2023-10-07T05:32:18.335Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30", size = 8321, upload-time = "2023-10-07T05:32:16.783Z" }, +] + +[[package]] +name = "decorator" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/60/8b/32f9823da46cde7df2087faa08cd98d01b908f8dcab982cdba9c84e85355/decorator-5.3.1.tar.gz", hash = "sha256:4cbcdd55a6efadb9dbea26b858f4fb3264567b52d69ca0d25b721b553f60ea82", size = 58084 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/7f/798705f5296a58ca505d600456748d1be48078eac8a7050d8a98bc9edb89/decorator-5.3.1-py3-none-any.whl", hash = "sha256:f47fe6fdbd2edd623ecfe36875d37aba411624e2670dd395dddae1358689bb3c", size = 10365 }, +] + +[[package]] +name = "dm-env" +version = "1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "absl-py" }, + { name = "dm-tree" }, + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/62/c9/93e8d6239d5806508a2ee4b370e67c6069943ca149f59f533923737a99b7/dm-env-1.6.tar.gz", hash = "sha256:a436eb1c654c39e0c986a516cee218bea7140b510fceff63f97eb4fcff3d93de", size = 20187 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/7e/36d548040e61337bf9182637a589c44da407a47a923ee88aec7f0e89867c/dm_env-1.6-py3-none-any.whl", hash = "sha256:0eabb6759dd453b625e041032f7ae0c1e87d4eb61b6a96b9ca586483837abf29", size = 26339 }, +] + +[[package]] +name = "dm-tree" +version = "0.1.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "absl-py" }, + { name = "attrs" }, + { name = "numpy" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/66/a3ec619d22b6baffa5ab853e8dc6ec9d0c837127948af59bb15b988d7312/dm_tree-0.1.10.tar.gz", hash = "sha256:22f37b599e01cc3402a17f79c257a802aebd8d326de05b54657650845956208a", size = 35748 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/dc/b01d0f70cde99b306731216a98287ba5926a50f27222f2ada0b99ad0911f/dm_tree-0.1.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8218af7b99701bb8b03001c82961dc2cf81d7a734958206d2ea1ede8fbbe2b5f", size = 314603 }, + { url = "https://files.pythonhosted.org/packages/40/72/3bafa58492862360113c1cccb26747c7863d417271e1572bacb3c281162f/dm_tree-0.1.10-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cacef6180fcfef30bab2cac5164e753e2f7a2e60e5da0feb81f2d318416f8d98", size = 182657 }, + { url = "https://files.pythonhosted.org/packages/78/10/587a2cdc05995069aa63b659d884eb3e58a3c86a5b4a00acdb7a316bddf3/dm_tree-0.1.10-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f0e8907bb6809dc195be3af077e382126eaebe06c00f835d09ae26e36d2165ff", size = 185008 }, + { url = "https://files.pythonhosted.org/packages/60/0e/08d938d84cbf791dde009b3d3a6637f27a0004235e700641a0ac038daac5/dm_tree-0.1.10-cp311-cp311-win_amd64.whl", hash = "sha256:a1c82dd4726a16ac6b6f7a77a5fb097ee396fd349ae301407eb5736f15b8fa16", size = 111472 }, +] + +[[package]] +name = "envpool" +version = "1.2.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dm-env" }, + { name = "envpool-assets" }, + { name = "envpool-assets-mujoco-large" }, + { name = "envpool-assets-mujoco-playground-humanoid" }, + { name = "glfw" }, + { name = "gymnasium" }, + { name = "matplotlib" }, + { name = "numpy" }, + { name = "optree" }, + { name = "packaging" }, + { name = "pillow" }, + { name = "types-protobuf" }, + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/6e/35/92b211319559188ce28ad6b01107549dbb304635b31f3298d4801cbd8cb1/envpool-1.2.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:44a7890a1367ae464f741f6f359ca8a8a3c70211ee3ff30290c0747fafd73156", size = 36721214 }, + { url = "https://files.pythonhosted.org/packages/1c/5f/ef37f239a8068d5f064ac019655210406338d638c4ded55e0d084f7fee4d/envpool-1.2.5-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c2edffc5e2982e1429b86cd3212a12f0c7d29cb91142d6d68a675b8b89d16a0a", size = 51151369 }, + { url = "https://files.pythonhosted.org/packages/a4/02/c28cf83f5d7731462d4612af19a9df9b793d6f0355a34b486fff6c4936ba/envpool-1.2.5-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3842a3df372b0f35db76d36b1c8dcc3c623566e4557031a232e7af5ab195e25", size = 52733750 }, + { url = "https://files.pythonhosted.org/packages/5a/74/37fa5d5ae0dbda100bc17eddb007c9346a57edd83176f99798637214d353/envpool-1.2.5-cp311-cp311-win_amd64.whl", hash = "sha256:0e28fbf613c563617a9575944f15735e0b585e875bf36d02298a5d1d89064f7a", size = 42490994 }, +] + +[[package]] +name = "envpool-assets" +version = "0.3.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/9f/29f93af34bbb7f1b333984096992be1307ebdafc7da6246f6efd132f6ba0/envpool_assets-0.3.0-py3-none-any.whl", hash = "sha256:b6f1d230a9f721d77cec41da8b3edb00a1db223eec8f826de919db8b9ac133e0", size = 62429187 }, +] + +[[package]] +name = "envpool-assets-mujoco-large" +version = "0.3.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/57/9c/c54565e6e1a608c42e1c176733d5054b73580e69bd911531252961e997df/envpool_assets_mujoco_large-0.3.0-py3-none-any.whl", hash = "sha256:43458cc529487d367f6195cfc71f6dee74fdd2e60ddc4ffa2143bf9bd29dd8a9", size = 89317713 }, +] + +[[package]] +name = "envpool-assets-mujoco-playground-humanoid" +version = "0.3.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/24/f192219df0a335a78b2bcb8a715499a687d1d8bf916e4eb32f0bf3bd6c4a/envpool_assets_mujoco_playground_humanoid-0.3.0-py3-none-any.whl", hash = "sha256:02ed9463918efd5620abe523c53c17a63f91797732e5301761d7037323972877", size = 91324818 }, +] + +[[package]] +name = "farama-notifications" +version = "0.0.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/91/14397890dde30adc4bee6462158933806207bc5dd10d7b4d09d5c33845cf/farama_notifications-0.0.6.tar.gz", hash = "sha256:b19acac4bb41d76e59e03394b5dd165f4761c86fa327f56307a35cbee3b60158", size = 2517, upload-time = "2026-04-24T08:43:57.603Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/f0/21f81892e4ed10f4ec3ef2e7cf8635fb76e7c0907c55d0da66be50094760/farama_notifications-0.0.6-py3-none-any.whl", hash = "sha256:f84839188efa1ce5bb361c2a84881b2dc2c0d0d7fb661ff00421820170930935", size = 2897, upload-time = "2026-04-24T08:43:56.785Z" }, +] + +[[package]] +name = "filelock" +version = "3.29.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b5/fe/997687a931ab51049acce6fa1f23e8f01216374ea81374ddee763c493db5/filelock-3.29.0.tar.gz", hash = "sha256:69974355e960702e789734cb4871f884ea6fe50bd8404051a3530bc07809cf90", size = 57571, upload-time = "2026-04-19T15:39:10.068Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/47/dd9a212ef6e343a6857485ffe25bba537304f1913bdbed446a23f7f592e1/filelock-3.29.0-py3-none-any.whl", hash = "sha256:96f5f6344709aa1572bbf631c640e4ebeeb519e08da902c39a001882f30ac258", size = 39812, upload-time = "2026-04-19T15:39:08.752Z" }, +] + +[[package]] +name = "fonttools" +version = "4.63.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/84/69/c97f2c18e0db87d2c7b15da1974dace76ae938f1cfa22e2727a648b7ed43/fonttools-4.63.0.tar.gz", hash = "sha256:caeb583deeb5168e694b65cda8b4ee62abedfa66cf88488734466f2366b9c4e0", size = 3597189, upload-time = "2026-05-14T12:04:30.958Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/75/2b/a7f1545bdf5da69c4bda0cea2a5781f0ad2a6623e0277267672db43c5fe6/fonttools-4.63.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2b8ae05d9eacf6081414d759c0a352769ac28ce31280d6bb8e77b03f9e3c449f", size = 2881793, upload-time = "2026-05-14T12:02:56.645Z" }, + { url = "https://files.pythonhosted.org/packages/49/50/965308c703f085f225db2886813b27e015b8b3438c350b22dd65b52c2a2c/fonttools-4.63.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:79cdc9f567aec74a72918fd060283911406750cbc9fd28c1316023deb6ce31a9", size = 2428130, upload-time = "2026-05-14T12:02:58.891Z" }, + { url = "https://files.pythonhosted.org/packages/d8/38/6937fbd7f2dc3a6b48725851bc2c15ec949b9af14d9bbcb5fe83cdf9bdf9/fonttools-4.63.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c14b4fd138c4bafcca294765c547914e1aa431ae1ca94ab99d8db08c958bd3b", size = 5111952, upload-time = "2026-05-14T12:03:01.263Z" }, + { url = "https://files.pythonhosted.org/packages/0b/43/a81f20050a3115b57d62c8e781446949512eac36690dc384ccea65ff4cc1/fonttools-4.63.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d76ac49f929aecaf82d83250b8347e099d7aecba0f4726c1d9b6df3b8bb5fe18", size = 5082308, upload-time = "2026-05-14T12:03:03.211Z" }, + { url = "https://files.pythonhosted.org/packages/67/00/cdd9d4944ca6ae280d01e69cc37bde3bf663630b837a6fc6d2cd65d80e0e/fonttools-4.63.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:dcf076a4474fe0d7367e5bbf5b052c7284fa1feca729c04176ce513521afd8a0", size = 5087932, upload-time = "2026-05-14T12:03:05.147Z" }, + { url = "https://files.pythonhosted.org/packages/f5/f1/0aa0dbea778c75adbef223c42019fd47d22262b905974d62d829545d485f/fonttools-4.63.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7dd683fef0663e9f0f45cf541d788d24caa3ec9db50796b588e1757d8b3bc007", size = 5213271, upload-time = "2026-05-14T12:03:07.238Z" }, + { url = "https://files.pythonhosted.org/packages/a8/99/253e4056e1f0e67b9390125a154b73b5eb73ad521bece95c004858fdeec2/fonttools-4.63.0-cp311-cp311-win32.whl", hash = "sha256:afefc1ed0a59785a7fb06ea7e1678e849c193e1e387db783579bc7b3056fcfcb", size = 2304473, upload-time = "2026-05-14T12:03:09.271Z" }, + { url = "https://files.pythonhosted.org/packages/08/60/defa5e69641db890a63be281f41345f4c33b157824eaf0b9fad3e08b0dcb/fonttools-4.63.0-cp311-cp311-win_amd64.whl", hash = "sha256:063e08bd17bd5a90127a14123de0d6a952dbc847695fd98b63c043d58057f90c", size = 2356389, upload-time = "2026-05-14T12:03:11.53Z" }, + { url = "https://files.pythonhosted.org/packages/2c/47/c99d5268f354002ce80f8d029cd9d7d872969da1de8b93d32de4dc56d6f4/fonttools-4.63.0-py3-none-any.whl", hash = "sha256:445af2eab030a16b9171ea8bdda7ebf7d96bda2df88ee182a464252f6e05e20d", size = 1164562, upload-time = "2026-05-14T12:04:29.092Z" }, +] + +[[package]] +name = "fsspec" +version = "2026.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d5/8d/1c51c094345df128ca4a990d633fe1a0ff28726c9e6b3c41ba65087bba1d/fsspec-2026.4.0.tar.gz", hash = "sha256:301d8ac70ae90ef3ad05dcf94d6c3754a097f9b5fe4667d2787aa359ec7df7e4", size = 312760, upload-time = "2026-04-29T20:42:38.635Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl", hash = "sha256:11ef7bb35dab8a394fde6e608221d5cf3e8499401c249bebaeaad760a1a8dec2", size = 203402, upload-time = "2026-04-29T20:42:36.842Z" }, +] + +[[package]] +name = "gitdb" +version = "4.0.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "smmap" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/94/63b0fc47eb32792c7ba1fe1b694daec9a63620db1e313033d18140c2320a/gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571", size = 394684, upload-time = "2025-01-02T07:20:46.413Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/61/5c78b91c3143ed5c14207f463aecfc8f9dbb5092fb2869baf37c273b2705/gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf", size = 62794, upload-time = "2025-01-02T07:20:43.624Z" }, +] + +[[package]] +name = "gitpython" +version = "3.1.50" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "gitdb" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/33/f6/354ae6491228b5eb40e10d89c4d13c651fe1cf7556e35ebdded50cff57ce/gitpython-3.1.50.tar.gz", hash = "sha256:80da2d12504d52e1f998772dc5baf6e553f8d2fcfe1fcc226c9d9a2ee3372dcc", size = 219798, upload-time = "2026-05-06T04:01:26.571Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/7a/1c6e3562dfd8950adbb11ffbc65d21e7c89d01a6e4f137fa981056de25c5/gitpython-3.1.50-py3-none-any.whl", hash = "sha256:d352abe2908d07355014abdd21ddf798c2a961469239afec4962e9da884858f9", size = 212507, upload-time = "2026-05-06T04:01:23.799Z" }, +] + +[[package]] +name = "glfw" +version = "2.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/96/72/642d4f12f61816ac96777f7360d413e3977a7dd08237d196f02da681b186/glfw-2.10.0.tar.gz", hash = "sha256:801e55d8581b34df9aa2cfea43feb06ff617576e2a8cc5dac23ee75b26d10abe", size = 31475 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/1f/a9ce08b1173b0ab625ee92f0c47a5278b3e76fd367699880d8ee7d56c338/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.p39.p310.p311.p312.p313-none-macosx_10_6_intel.whl", hash = "sha256:5f365a8c94bcea71ec91327e7c16e7cf739128479a18b8c1241b004b40acc412", size = 105329 }, + { url = "https://files.pythonhosted.org/packages/7c/96/5a2220abcbd027eebcf8bedd28207a2de168899e51be13ba01ebdd4147a1/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.p39.p310.p311.p312.p313-none-macosx_11_0_arm64.whl", hash = "sha256:5328db1a92d07abd988730517ec02aa8390d3e6ef7ce98c8b57ecba2f43a39ba", size = 102179 }, + { url = "https://files.pythonhosted.org/packages/9d/41/a5bd1d9e1808f400102bd7d328c4ac17b65fb2fc8014014ec6f23d02f662/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.p39.p310.p311.p312.p313-none-manylinux2014_aarch64.whl", hash = "sha256:312c4c1dd5509613ed6bc1e95a8dbb75a36b6dcc4120f50dc3892b40172e9053", size = 230039 }, + { url = "https://files.pythonhosted.org/packages/80/aa/3b503c448609dee6cb4e7138b4109338f0e65b97be107ab85562269d378d/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.p39.p310.p311.p312.p313-none-manylinux2014_x86_64.whl", hash = "sha256:59c53387dc08c62e8bed86bbe3a8d53ab1b27161281ffa0e7f27b64284e2627c", size = 241984 }, + { url = "https://files.pythonhosted.org/packages/ac/2d/bfe39a42cad8e80b02bf5f7cae19ba67832c1810bbd3624a8e83153d74a4/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.p39.p310.p311.p312.p313-none-manylinux_2_28_aarch64.whl", hash = "sha256:c6f292fdaf3f9a99e598ede6582d21c523a6f51f8f5e66213849101a6bcdc699", size = 231052 }, + { url = "https://files.pythonhosted.org/packages/f7/02/6e639e90f181dc9127046e00d0528f9f7ad12d428972e3a5378b9aefdb0b/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.p39.p310.p311.p312.p313-none-manylinux_2_28_x86_64.whl", hash = "sha256:7916034efa867927892635733a3b6af8cd95ceb10566fd7f1e0d2763c2ee8b12", size = 243525 }, + { url = "https://files.pythonhosted.org/packages/84/06/cb588ca65561defe0fc48d1df4c2ac12569b81231ae4f2b52ab37007d0bd/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.p39.p310.p311.p312.p313-none-win32.whl", hash = "sha256:6c9549da71b93e367b4d71438798daae1da2592039fd14204a80a1a2348ae127", size = 552685 }, + { url = "https://files.pythonhosted.org/packages/86/27/00c9c96af18ac0a5eac2ff61cbe306551a2d770d7173f396d0792ee1a59e/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.p39.p310.p311.p312.p313-none-win_amd64.whl", hash = "sha256:6292d5d6634d668cd23d337e6089491d3945a9aa4ac6e1667b0003520d7caa51", size = 559466 }, + { url = "https://files.pythonhosted.org/packages/b3/87/de0b33f6f00687499ca1371f22aa73396341b85bf88f1a284f9da8842493/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.py39.py310.py311.py312.py313.py314-none-macosx_10_6_intel.whl", hash = "sha256:2aab89d2d9535635ba011fc7303390685169a1aa6731ad580d08d043524b8899", size = 105326 }, + { url = "https://files.pythonhosted.org/packages/b6/a6/6ea2f73ad4474896d9e38b3ffbe6ffd5a802c738392269e99e8c6621a461/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.py39.py310.py311.py312.py313.py314-none-macosx_11_0_arm64.whl", hash = "sha256:23936202a107039b5372f0b88ae1d11080746aa1c78910a45d4a0c4cf408cfaa", size = 102180 }, + { url = "https://files.pythonhosted.org/packages/58/19/d81b19e8261b9cb51b81d1402167791fef81088dfe91f0c4e9d136fdc5ca/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.py39.py310.py311.py312.py313.py314-none-manylinux2014_aarch64.whl", hash = "sha256:7be06d0838f61df67bd54cb6266a6193d54083acb3624ff3c3812a6358406fa4", size = 230038 }, + { url = "https://files.pythonhosted.org/packages/e2/fa/b035636cd82198b97b51a93efe9cfc4343d6b15cefbd336a3f2be871d848/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.py39.py310.py311.py312.py313.py314-none-manylinux2014_x86_64.whl", hash = "sha256:91d36b3582a766512eff8e3b5dcc2d3ffcbf10b7cf448551085a08a10f1b8244", size = 241983 }, + { url = "https://files.pythonhosted.org/packages/ff/b4/f7b6cc022dd7c68b6c702d19da5d591f978f89c958b9bd3090615db0c739/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.py39.py310.py311.py312.py313.py314-none-manylinux_2_28_aarch64.whl", hash = "sha256:27c9e9a2d5e1dc3c9e3996171d844d9df9a5a101e797cb94cce217b7afcf8fd9", size = 231053 }, + { url = "https://files.pythonhosted.org/packages/5a/3f/efeb7c6801c46e11bd666a5180f0d615f74f72264212f74f39586c6fda9d/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.py39.py310.py311.py312.py313.py314-none-manylinux_2_28_x86_64.whl", hash = "sha256:ce6724bb7cb3d0543dcba17206dce909f94176e68220b8eafee72e9f92bcf542", size = 243522 }, + { url = "https://files.pythonhosted.org/packages/cf/b9/b04c3aa0aad2870cfe799f32f8b59789c98e1816bbce9e83f4823c5b840b/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.py39.py310.py311.py312.py313.py314-none-win32.whl", hash = "sha256:fca724a21a372731edb290841edd28a9fb1ee490f833392752844ac807c0086a", size = 552682 }, + { url = "https://files.pythonhosted.org/packages/bd/e1/6d6816b296a529ac9b897ad228b1e084eb1f92319e96371880eebdc874a6/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.py39.py310.py311.py312.py313.py314-none-win_amd64.whl", hash = "sha256:823c0bd7770977d4b10e0ed0aef2f3682276b7c88b8b65cfc540afce5951392f", size = 559464 }, + { url = "https://files.pythonhosted.org/packages/8c/a8/d4dab8a58fc2e6981fc7a58c4e56ba9d777fb24931cec6a22152edbb3540/glfw-2.10.0-py2.py3-none-macosx_10_6_intel.whl", hash = "sha256:a0d1f29f206219cc291edfb6cace663a86da2470632551c998e3db82d48ea177", size = 105288 }, + { url = "https://files.pythonhosted.org/packages/14/61/68d35e001872a7705112418da236fa2418d4f2e5419f8b2837f9b81bb3da/glfw-2.10.0-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:d28d6f3ef217e64e35dc6fd0a7acb4cec9bfe7cd14dd9b35a7228a87002de154", size = 102139 }, + { url = "https://files.pythonhosted.org/packages/4e/e1/ca5984081aaae07c9d371cb11dc4e4ff603510678ed9b73e58b6c351fe63/glfw-2.10.0-py2.py3-none-manylinux2014_aarch64.whl", hash = "sha256:f968b522bb6a0e04aaf4dcac30a476d7229308bb2bac406a60587debb5a61e29", size = 229998 }, + { url = "https://files.pythonhosted.org/packages/fa/c4/82ac75fdcfba2896da7a573c0fc7f8ceb8f77ead6866d500d06c32f1c464/glfw-2.10.0-py2.py3-none-manylinux2014_x86_64.whl", hash = "sha256:68cf3752bdadb6f4bc0a876247c28c88c7251ac39f8af076ed938fdfd71e72dd", size = 241944 }, + { url = "https://files.pythonhosted.org/packages/e3/96/9f691823cca5eb6a08f346bd0ff03b78032db9370b509a1e9c8976fb20a5/glfw-2.10.0-py2.py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:44d98de5dbf8f727e0cb29f9b29d29528ea7570f2e6f42f8430a69df05f12b48", size = 231009 }, + { url = "https://files.pythonhosted.org/packages/3f/93/977b9e679e356871d428ae7a1139ec767dd5177bed58a6344b4d2199e00f/glfw-2.10.0-py2.py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:cca5158d62189e08792b1ae54f92307a282921a0e7783315b467e21b0a381c88", size = 243480 }, + { url = "https://files.pythonhosted.org/packages/f9/bd/cea9569c8f2188b0a104472951420434a3e1f5cf26f5836ef9d7227a1a30/glfw-2.10.0-py2.py3-none-win32.whl", hash = "sha256:5e024509989740e8e7b86cc4aab508195495f79879072b0e1f68bd036a2916ad", size = 552641 }, + { url = "https://files.pythonhosted.org/packages/cc/9b/4366ad3e1c0688146c70aa6143584d6a8d88583b9390f106250e25a3d5cd/glfw-2.10.0-py2.py3-none-win_amd64.whl", hash = "sha256:7f787ee8645781f10e8800438ce4357ab38c573ffb191aba380c1e72eba6311c", size = 559423 }, +] + +[[package]] +name = "glfw" +version = "2.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/96/72/642d4f12f61816ac96777f7360d413e3977a7dd08237d196f02da681b186/glfw-2.10.0.tar.gz", hash = "sha256:801e55d8581b34df9aa2cfea43feb06ff617576e2a8cc5dac23ee75b26d10abe", size = 31475 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/1f/a9ce08b1173b0ab625ee92f0c47a5278b3e76fd367699880d8ee7d56c338/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.p39.p310.p311.p312.p313-none-macosx_10_6_intel.whl", hash = "sha256:5f365a8c94bcea71ec91327e7c16e7cf739128479a18b8c1241b004b40acc412", size = 105329 }, + { url = "https://files.pythonhosted.org/packages/7c/96/5a2220abcbd027eebcf8bedd28207a2de168899e51be13ba01ebdd4147a1/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.p39.p310.p311.p312.p313-none-macosx_11_0_arm64.whl", hash = "sha256:5328db1a92d07abd988730517ec02aa8390d3e6ef7ce98c8b57ecba2f43a39ba", size = 102179 }, + { url = "https://files.pythonhosted.org/packages/9d/41/a5bd1d9e1808f400102bd7d328c4ac17b65fb2fc8014014ec6f23d02f662/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.p39.p310.p311.p312.p313-none-manylinux2014_aarch64.whl", hash = "sha256:312c4c1dd5509613ed6bc1e95a8dbb75a36b6dcc4120f50dc3892b40172e9053", size = 230039 }, + { url = "https://files.pythonhosted.org/packages/80/aa/3b503c448609dee6cb4e7138b4109338f0e65b97be107ab85562269d378d/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.p39.p310.p311.p312.p313-none-manylinux2014_x86_64.whl", hash = "sha256:59c53387dc08c62e8bed86bbe3a8d53ab1b27161281ffa0e7f27b64284e2627c", size = 241984 }, + { url = "https://files.pythonhosted.org/packages/ac/2d/bfe39a42cad8e80b02bf5f7cae19ba67832c1810bbd3624a8e83153d74a4/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.p39.p310.p311.p312.p313-none-manylinux_2_28_aarch64.whl", hash = "sha256:c6f292fdaf3f9a99e598ede6582d21c523a6f51f8f5e66213849101a6bcdc699", size = 231052 }, + { url = "https://files.pythonhosted.org/packages/f7/02/6e639e90f181dc9127046e00d0528f9f7ad12d428972e3a5378b9aefdb0b/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.p39.p310.p311.p312.p313-none-manylinux_2_28_x86_64.whl", hash = "sha256:7916034efa867927892635733a3b6af8cd95ceb10566fd7f1e0d2763c2ee8b12", size = 243525 }, + { url = "https://files.pythonhosted.org/packages/84/06/cb588ca65561defe0fc48d1df4c2ac12569b81231ae4f2b52ab37007d0bd/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.p39.p310.p311.p312.p313-none-win32.whl", hash = "sha256:6c9549da71b93e367b4d71438798daae1da2592039fd14204a80a1a2348ae127", size = 552685 }, + { url = "https://files.pythonhosted.org/packages/86/27/00c9c96af18ac0a5eac2ff61cbe306551a2d770d7173f396d0792ee1a59e/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.p39.p310.p311.p312.p313-none-win_amd64.whl", hash = "sha256:6292d5d6634d668cd23d337e6089491d3945a9aa4ac6e1667b0003520d7caa51", size = 559466 }, + { url = "https://files.pythonhosted.org/packages/b3/87/de0b33f6f00687499ca1371f22aa73396341b85bf88f1a284f9da8842493/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.py39.py310.py311.py312.py313.py314-none-macosx_10_6_intel.whl", hash = "sha256:2aab89d2d9535635ba011fc7303390685169a1aa6731ad580d08d043524b8899", size = 105326 }, + { url = "https://files.pythonhosted.org/packages/b6/a6/6ea2f73ad4474896d9e38b3ffbe6ffd5a802c738392269e99e8c6621a461/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.py39.py310.py311.py312.py313.py314-none-macosx_11_0_arm64.whl", hash = "sha256:23936202a107039b5372f0b88ae1d11080746aa1c78910a45d4a0c4cf408cfaa", size = 102180 }, + { url = "https://files.pythonhosted.org/packages/58/19/d81b19e8261b9cb51b81d1402167791fef81088dfe91f0c4e9d136fdc5ca/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.py39.py310.py311.py312.py313.py314-none-manylinux2014_aarch64.whl", hash = "sha256:7be06d0838f61df67bd54cb6266a6193d54083acb3624ff3c3812a6358406fa4", size = 230038 }, + { url = "https://files.pythonhosted.org/packages/e2/fa/b035636cd82198b97b51a93efe9cfc4343d6b15cefbd336a3f2be871d848/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.py39.py310.py311.py312.py313.py314-none-manylinux2014_x86_64.whl", hash = "sha256:91d36b3582a766512eff8e3b5dcc2d3ffcbf10b7cf448551085a08a10f1b8244", size = 241983 }, + { url = "https://files.pythonhosted.org/packages/ff/b4/f7b6cc022dd7c68b6c702d19da5d591f978f89c958b9bd3090615db0c739/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.py39.py310.py311.py312.py313.py314-none-manylinux_2_28_aarch64.whl", hash = "sha256:27c9e9a2d5e1dc3c9e3996171d844d9df9a5a101e797cb94cce217b7afcf8fd9", size = 231053 }, + { url = "https://files.pythonhosted.org/packages/5a/3f/efeb7c6801c46e11bd666a5180f0d615f74f72264212f74f39586c6fda9d/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.py39.py310.py311.py312.py313.py314-none-manylinux_2_28_x86_64.whl", hash = "sha256:ce6724bb7cb3d0543dcba17206dce909f94176e68220b8eafee72e9f92bcf542", size = 243522 }, + { url = "https://files.pythonhosted.org/packages/cf/b9/b04c3aa0aad2870cfe799f32f8b59789c98e1816bbce9e83f4823c5b840b/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.py39.py310.py311.py312.py313.py314-none-win32.whl", hash = "sha256:fca724a21a372731edb290841edd28a9fb1ee490f833392752844ac807c0086a", size = 552682 }, + { url = "https://files.pythonhosted.org/packages/bd/e1/6d6816b296a529ac9b897ad228b1e084eb1f92319e96371880eebdc874a6/glfw-2.10.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.py39.py310.py311.py312.py313.py314-none-win_amd64.whl", hash = "sha256:823c0bd7770977d4b10e0ed0aef2f3682276b7c88b8b65cfc540afce5951392f", size = 559464 }, + { url = "https://files.pythonhosted.org/packages/8c/a8/d4dab8a58fc2e6981fc7a58c4e56ba9d777fb24931cec6a22152edbb3540/glfw-2.10.0-py2.py3-none-macosx_10_6_intel.whl", hash = "sha256:a0d1f29f206219cc291edfb6cace663a86da2470632551c998e3db82d48ea177", size = 105288 }, + { url = "https://files.pythonhosted.org/packages/14/61/68d35e001872a7705112418da236fa2418d4f2e5419f8b2837f9b81bb3da/glfw-2.10.0-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:d28d6f3ef217e64e35dc6fd0a7acb4cec9bfe7cd14dd9b35a7228a87002de154", size = 102139 }, + { url = "https://files.pythonhosted.org/packages/4e/e1/ca5984081aaae07c9d371cb11dc4e4ff603510678ed9b73e58b6c351fe63/glfw-2.10.0-py2.py3-none-manylinux2014_aarch64.whl", hash = "sha256:f968b522bb6a0e04aaf4dcac30a476d7229308bb2bac406a60587debb5a61e29", size = 229998 }, + { url = "https://files.pythonhosted.org/packages/fa/c4/82ac75fdcfba2896da7a573c0fc7f8ceb8f77ead6866d500d06c32f1c464/glfw-2.10.0-py2.py3-none-manylinux2014_x86_64.whl", hash = "sha256:68cf3752bdadb6f4bc0a876247c28c88c7251ac39f8af076ed938fdfd71e72dd", size = 241944 }, + { url = "https://files.pythonhosted.org/packages/e3/96/9f691823cca5eb6a08f346bd0ff03b78032db9370b509a1e9c8976fb20a5/glfw-2.10.0-py2.py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:44d98de5dbf8f727e0cb29f9b29d29528ea7570f2e6f42f8430a69df05f12b48", size = 231009 }, + { url = "https://files.pythonhosted.org/packages/3f/93/977b9e679e356871d428ae7a1139ec767dd5177bed58a6344b4d2199e00f/glfw-2.10.0-py2.py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:cca5158d62189e08792b1ae54f92307a282921a0e7783315b467e21b0a381c88", size = 243480 }, + { url = "https://files.pythonhosted.org/packages/f9/bd/cea9569c8f2188b0a104472951420434a3e1f5cf26f5836ef9d7227a1a30/glfw-2.10.0-py2.py3-none-win32.whl", hash = "sha256:5e024509989740e8e7b86cc4aab508195495f79879072b0e1f68bd036a2916ad", size = 552641 }, + { url = "https://files.pythonhosted.org/packages/cc/9b/4366ad3e1c0688146c70aa6143584d6a8d88583b9390f106250e25a3d5cd/glfw-2.10.0-py2.py3-none-win_amd64.whl", hash = "sha256:7f787ee8645781f10e8800438ce4357ab38c573ffb191aba380c1e72eba6311c", size = 559423 }, +] + +[[package]] +name = "gymnasium" +version = "1.2.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cloudpickle" }, + { name = "farama-notifications" }, + { name = "numpy" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/59/653a9417d98ed3e29ef9734ba52c3495f6c6823b8d5c0c75369f25111708/gymnasium-1.2.3.tar.gz", hash = "sha256:2b2cb5b5fbbbdf3afb9f38ca952cc48aa6aa3e26561400d940747fda3ad42509", size = 829230, upload-time = "2025-12-18T16:51:10.234Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/d3/ea5f088e3638dbab12e5c20d6559d5b3bdaeaa1f2af74e526e6815836285/gymnasium-1.2.3-py3-none-any.whl", hash = "sha256:e6314bba8f549c7fdcc8677f7cd786b64908af6e79b57ddaa5ce1825bffb5373", size = 952113, upload-time = "2025-12-18T16:51:08.445Z" }, +] + +[package.optional-dependencies] +atari = [ + { name = "ale-py" }, +] + +[[package]] +name = "idna" +version = "3.15" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/77/7b3966d0b9d1d31a36ddf1746926a11dface89a83409bf1483f0237aa758/idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc", size = 199245, upload-time = "2026-05-12T22:45:57.011Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" }, +] + +[[package]] +name = "imageio" +version = "2.37.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "pillow" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/84/93bcd1300216ea50811cee96873b84a1bebf8d0489ffaf7f2a3756bab866/imageio-2.37.3.tar.gz", hash = "sha256:bbb37efbfc4c400fcd534b367b91fcd66d5da639aaa138034431a1c5e0a41451", size = 389673 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/fa/391e437a34e55095173dca5f24070d89cbc233ff85bf1c29c93248c6588d/imageio-2.37.3-py3-none-any.whl", hash = "sha256:46f5bb8522cd421c0f5ae104d8268f569d856b29eb1a13b92829d1970f32c9f0", size = 317646 }, +] + +[[package]] +name = "imageio-ffmpeg" +version = "0.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/44/bd/c3343c721f2a1b0c9fc71c1aebf1966a3b7f08c2eea8ed5437a2865611d6/imageio_ffmpeg-0.6.0.tar.gz", hash = "sha256:e2556bed8e005564a9f925bb7afa4002d82770d6b08825078b7697ab88ba1755", size = 25210 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/58/87ef68ac83f4c7690961bce288fd8e382bc5f1513860fc7f90a9c1c1c6bf/imageio_ffmpeg-0.6.0-py3-none-macosx_10_9_intel.macosx_10_9_x86_64.whl", hash = "sha256:9d2baaf867088508d4a3458e61eeb30e945c4ad8016025545f66c4b5aaef0a61", size = 24932969 }, + { url = "https://files.pythonhosted.org/packages/40/5c/f3d8a657d362cc93b81aab8feda487317da5b5d31c0e1fdfd5e986e55d17/imageio_ffmpeg-0.6.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:b1ae3173414b5fc5f538a726c4e48ea97edc0d2cdc11f103afee655c463fa742", size = 21113891 }, + { url = "https://files.pythonhosted.org/packages/33/e7/1925bfbc563c39c1d2e82501d8372734a5c725e53ac3b31b4c2d081e895b/imageio_ffmpeg-0.6.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:1d47bebd83d2c5fc770720d211855f208af8a596c82d17730aa51e815cdee6dc", size = 25632706 }, + { url = "https://files.pythonhosted.org/packages/a0/2d/43c8522a2038e9d0e7dbdf3a61195ecc31ca576fb1527a528c877e87d973/imageio_ffmpeg-0.6.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:c7e46fcec401dd990405049d2e2f475e2b397779df2519b544b8aab515195282", size = 29498237 }, + { url = "https://files.pythonhosted.org/packages/a0/13/59da54728351883c3c1d9fca1710ab8eee82c7beba585df8f25ca925f08f/imageio_ffmpeg-0.6.0-py3-none-win32.whl", hash = "sha256:196faa79366b4a82f95c0f4053191d2013f4714a715780f0ad2a68ff37483cc2", size = 19652251 }, + { url = "https://files.pythonhosted.org/packages/2c/c6/fa760e12a2483469e2bf5058c5faff664acf66cadb4df2ad6205b016a73d/imageio_ffmpeg-0.6.0-py3-none-win_amd64.whl", hash = "sha256:02fa47c83703c37df6bfe4896aab339013f62bf02c5ebf2dce6da56af04ffc0a", size = 31246824 }, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + +[[package]] +name = "kiwisolver" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/67/9c61eccb13f0bdca9307614e782fec49ffdde0f7a2314935d489fa93cd9c/kiwisolver-1.5.0.tar.gz", hash = "sha256:d4193f3d9dc3f6f79aaed0e5637f45d98850ebf01f7ca20e69457f3e8946b66a", size = 103482, upload-time = "2026-03-09T13:15:53.382Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/dd/a495a9c104be1c476f0386e714252caf2b7eca883915422a64c50b88c6f5/kiwisolver-1.5.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9eed0f7edbb274413b6ee781cca50541c8c0facd3d6fd289779e494340a2b85c", size = 122798, upload-time = "2026-03-09T13:12:58.963Z" }, + { url = "https://files.pythonhosted.org/packages/11/60/37b4047a2af0cf5ef6d8b4b26e91829ae6fc6a2d1f74524bcb0e7cd28a32/kiwisolver-1.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3c4923e404d6bcd91b6779c009542e5647fef32e4a5d75e115e3bbac6f2335eb", size = 66216, upload-time = "2026-03-09T13:13:00.155Z" }, + { url = "https://files.pythonhosted.org/packages/0a/aa/510dc933d87767584abfe03efa445889996c70c2990f6f87c3ebaa0a18c5/kiwisolver-1.5.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0df54df7e686afa55e6f21fb86195224a6d9beb71d637e8d7920c95cf0f89aac", size = 63911, upload-time = "2026-03-09T13:13:01.671Z" }, + { url = "https://files.pythonhosted.org/packages/80/46/bddc13df6c2a40741e0cc7865bb1c9ed4796b6760bd04ce5fae3928ef917/kiwisolver-1.5.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2517e24d7315eb51c10664cdb865195df38ab74456c677df67bb47f12d088a27", size = 1438209, upload-time = "2026-03-09T13:13:03.385Z" }, + { url = "https://files.pythonhosted.org/packages/fd/d6/76621246f5165e5372f02f5e6f3f48ea336a8f9e96e43997d45b240ed8cd/kiwisolver-1.5.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff710414307fefa903e0d9bdf300972f892c23477829f49504e59834f4195398", size = 1248888, upload-time = "2026-03-09T13:13:05.231Z" }, + { url = "https://files.pythonhosted.org/packages/b2/c1/31559ec6fb39a5b48035ce29bb63ade628f321785f38c384dee3e2c08bc1/kiwisolver-1.5.0-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6176c1811d9d5a04fa391c490cc44f451e240697a16977f11c6f722efb9041db", size = 1266304, upload-time = "2026-03-09T13:13:06.743Z" }, + { url = "https://files.pythonhosted.org/packages/5e/ef/1cb8276f2d29cc6a41e0a042f27946ca347d3a4a75acf85d0a16aa6dcc82/kiwisolver-1.5.0-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50847dca5d197fcbd389c805aa1a1cf32f25d2e7273dc47ab181a517666b68cc", size = 1319650, upload-time = "2026-03-09T13:13:08.607Z" }, + { url = "https://files.pythonhosted.org/packages/4c/e4/5ba3cecd7ce6236ae4a80f67e5d5531287337d0e1f076ca87a5abe4cd5d0/kiwisolver-1.5.0-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:01808c6d15f4c3e8559595d6d1fe6411c68e4a3822b4b9972b44473b24f4e679", size = 970949, upload-time = "2026-03-09T13:13:10.299Z" }, + { url = "https://files.pythonhosted.org/packages/5a/69/dc61f7ae9a2f071f26004ced87f078235b5507ab6e5acd78f40365655034/kiwisolver-1.5.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:f1f9f4121ec58628c96baa3de1a55a4e3a333c5102c8e94b64e23bf7b2083309", size = 2199125, upload-time = "2026-03-09T13:13:11.841Z" }, + { url = "https://files.pythonhosted.org/packages/e5/7b/abbe0f1b5afa85f8d084b73e90e5f801c0939eba16ac2e49af7c61a6c28d/kiwisolver-1.5.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:b7d335370ae48a780c6e6a6bbfa97342f563744c39c35562f3f367665f5c1de2", size = 2293783, upload-time = "2026-03-09T13:13:14.399Z" }, + { url = "https://files.pythonhosted.org/packages/8a/80/5908ae149d96d81580d604c7f8aefd0e98f4fd728cf172f477e9f2a81744/kiwisolver-1.5.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:800ee55980c18545af444d93fdd60c56b580db5cc54867d8cbf8a1dc0829938c", size = 1960726, upload-time = "2026-03-09T13:13:16.047Z" }, + { url = "https://files.pythonhosted.org/packages/84/08/a78cb776f8c085b7143142ce479859cfec086bd09ee638a317040b6ef420/kiwisolver-1.5.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:c438f6ca858697c9ab67eb28246c92508af972e114cac34e57a6d4ba17a3ac08", size = 2464738, upload-time = "2026-03-09T13:13:17.897Z" }, + { url = "https://files.pythonhosted.org/packages/b1/e1/65584da5356ed6cb12c63791a10b208860ac40a83de165cb6a6751a686e3/kiwisolver-1.5.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:8c63c91f95173f9c2a67c7c526b2cea976828a0e7fced9cdcead2802dc10f8a4", size = 2270718, upload-time = "2026-03-09T13:13:19.421Z" }, + { url = "https://files.pythonhosted.org/packages/be/6c/28f17390b62b8f2f520e2915095b3c94d88681ecf0041e75389d9667f202/kiwisolver-1.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:beb7f344487cdcb9e1efe4b7a29681b74d34c08f0043a327a74da852a6749e7b", size = 73480, upload-time = "2026-03-09T13:13:20.818Z" }, + { url = "https://files.pythonhosted.org/packages/d8/0e/2ee5debc4f77a625778fec5501ff3e8036fe361b7ee28ae402a485bb9694/kiwisolver-1.5.0-cp311-cp311-win_arm64.whl", hash = "sha256:ad4ae4ffd1ee9cd11357b4c66b612da9888f4f4daf2f36995eda64bd45370cac", size = 64930, upload-time = "2026-03-09T13:13:21.997Z" }, + { url = "https://files.pythonhosted.org/packages/e9/eb/5fcbbbf9a0e2c3a35effb88831a483345326bbc3a030a3b5b69aee647f84/kiwisolver-1.5.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:ec4c85dc4b687c7f7f15f553ff26a98bfe8c58f5f7f0ac8905f0ba4c7be60232", size = 59532, upload-time = "2026-03-09T13:15:47.047Z" }, + { url = "https://files.pythonhosted.org/packages/c3/9b/e17104555bb4db148fd52327feea1e96be4b88e8e008b029002c281a21ab/kiwisolver-1.5.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:12e91c215a96e39f57989c8912ae761286ac5a9584d04030ceb3368a357f017a", size = 57420, upload-time = "2026-03-09T13:15:48.199Z" }, + { url = "https://files.pythonhosted.org/packages/48/44/2b5b95b7aa39fb2d8d9d956e0f3d5d45aef2ae1d942d4c3ffac2f9cfed1a/kiwisolver-1.5.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:be4a51a55833dc29ab5d7503e7bcb3b3af3402d266018137127450005cdfe737", size = 79892, upload-time = "2026-03-09T13:15:49.694Z" }, + { url = "https://files.pythonhosted.org/packages/52/7d/7157f9bba6b455cfb4632ed411e199fc8b8977642c2b12082e1bd9e6d173/kiwisolver-1.5.0-pp311-pypy311_pp73-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:daae526907e262de627d8f70058a0f64acc9e2641c164c99c8f594b34a799a16", size = 77603, upload-time = "2026-03-09T13:15:50.945Z" }, + { url = "https://files.pythonhosted.org/packages/0a/dd/8050c947d435c8d4bc94e3252f4d8bb8a76cfb424f043a8680be637a57f1/kiwisolver-1.5.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:59cd8683f575d96df5bb48f6add94afc055012c29e28124fcae2b63661b9efb1", size = 73558, upload-time = "2026-03-09T13:15:52.112Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/db/fefacb2136439fc8dd20e797950e749aa1f4997ed584c62cfb8ef7c2be0e/markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad", size = 11631, upload-time = "2025-09-27T18:36:18.185Z" }, + { url = "https://files.pythonhosted.org/packages/e1/2e/5898933336b61975ce9dc04decbc0a7f2fee78c30353c5efba7f2d6ff27a/markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a", size = 12058, upload-time = "2025-09-27T18:36:19.444Z" }, + { url = "https://files.pythonhosted.org/packages/1d/09/adf2df3699d87d1d8184038df46a9c80d78c0148492323f4693df54e17bb/markupsafe-3.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50", size = 24287, upload-time = "2025-09-27T18:36:20.768Z" }, + { url = "https://files.pythonhosted.org/packages/30/ac/0273f6fcb5f42e314c6d8cd99effae6a5354604d461b8d392b5ec9530a54/markupsafe-3.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf", size = 22940, upload-time = "2025-09-27T18:36:22.249Z" }, + { url = "https://files.pythonhosted.org/packages/19/ae/31c1be199ef767124c042c6c3e904da327a2f7f0cd63a0337e1eca2967a8/markupsafe-3.0.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f", size = 21887, upload-time = "2025-09-27T18:36:23.535Z" }, + { url = "https://files.pythonhosted.org/packages/b2/76/7edcab99d5349a4532a459e1fe64f0b0467a3365056ae550d3bcf3f79e1e/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a", size = 23692, upload-time = "2025-09-27T18:36:24.823Z" }, + { url = "https://files.pythonhosted.org/packages/a4/28/6e74cdd26d7514849143d69f0bf2399f929c37dc2b31e6829fd2045b2765/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115", size = 21471, upload-time = "2025-09-27T18:36:25.95Z" }, + { url = "https://files.pythonhosted.org/packages/62/7e/a145f36a5c2945673e590850a6f8014318d5577ed7e5920a4b3448e0865d/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a", size = 22923, upload-time = "2025-09-27T18:36:27.109Z" }, + { url = "https://files.pythonhosted.org/packages/0f/62/d9c46a7f5c9adbeeeda52f5b8d802e1094e9717705a645efc71b0913a0a8/markupsafe-3.0.3-cp311-cp311-win32.whl", hash = "sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19", size = 14572, upload-time = "2025-09-27T18:36:28.045Z" }, + { url = "https://files.pythonhosted.org/packages/83/8a/4414c03d3f891739326e1783338e48fb49781cc915b2e0ee052aa490d586/markupsafe-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01", size = 15077, upload-time = "2025-09-27T18:36:29.025Z" }, + { url = "https://files.pythonhosted.org/packages/35/73/893072b42e6862f319b5207adc9ae06070f095b358655f077f69a35601f0/markupsafe-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c", size = 13876, upload-time = "2025-09-27T18:36:29.954Z" }, +] + +[[package]] +name = "matplotlib" +version = "3.10.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "contourpy" }, + { name = "cycler" }, + { name = "fonttools" }, + { name = "kiwisolver" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pillow" }, + { name = "pyparsing" }, + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/63/1b/4be5be87d43d327a0cf4de1a56e86f7f84c89312452406cf122efe2839e6/matplotlib-3.10.9.tar.gz", hash = "sha256:fd66508e8c6877d98e586654b608a0456db8d7e8a546eb1e2600efd957302358", size = 34811233, upload-time = "2026-04-24T00:14:13.539Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4c/8c/290f021104741fea63769c31494f5324c0cd249bf536a65a4350767b1f22/matplotlib-3.10.9-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:68cfdcede415f7c8f5577b03303dd94526cdb6d11036cecdc205e08733b2d2bb", size = 8306860, upload-time = "2026-04-24T00:12:01.207Z" }, + { url = "https://files.pythonhosted.org/packages/51/18/325cd32ece1120d1da51cc4e4294c6580190699490183fc2fe8cb6d61ec5/matplotlib-3.10.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dfca0129678bd56379db26c52b5d77ed7de314c047492fbdc763aa7501710cfb", size = 8199254, upload-time = "2026-04-24T00:12:04.239Z" }, + { url = "https://files.pythonhosted.org/packages/79/db/e28c1b83e3680740aa78925f5fb2ae4d16207207419ad75ea9fe604f8676/matplotlib-3.10.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8e436d155fa8a3399dc62683f8f5d0e2e50d25d0144a73edd73f82eec8f4abfb", size = 8777092, upload-time = "2026-04-24T00:12:06.793Z" }, + { url = "https://files.pythonhosted.org/packages/55/fa/3ce7adfe9ba101748f465211660d9c6374c876b671bdb8c2bb6d347e8b94/matplotlib-3.10.9-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:56fc0bd271b00025c6edfdc7c2dcd247372c8e1544971d62e1dc7c17367e8bf9", size = 9595691, upload-time = "2026-04-24T00:12:09.706Z" }, + { url = "https://files.pythonhosted.org/packages/36/c4/6960a76686ed668f2c60f84e9799ba4c0d56abdb36b1577b60c1d061d1ec/matplotlib-3.10.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a5a6104ed666402ba5106d7f36e0e0cdca4e8d7fa4d39708ca88019e2835a2eb", size = 9659771, upload-time = "2026-04-24T00:12:12.766Z" }, + { url = "https://files.pythonhosted.org/packages/7e/0d/271aace3342157c64700c9ff4c59c7b392f3dbab393692e8db6fbe7ab96c/matplotlib-3.10.9-cp311-cp311-win_amd64.whl", hash = "sha256:d730e984eddf56974c3e72b6129c7ca462ac38dc624338f4b0b23eb23ecba00f", size = 8205112, upload-time = "2026-04-24T00:12:15.773Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ee/cb57ad4754f3e7b9174ce6ce66d9205fb827067e48a9f58ac09d7e7d6b77/matplotlib-3.10.9-cp311-cp311-win_arm64.whl", hash = "sha256:51bf0ddbdc598e060d46c16b5590708f81a1624cefbaaf62f6a81bf9285b8c80", size = 8132310, upload-time = "2026-04-24T00:12:18.645Z" }, + { url = "https://files.pythonhosted.org/packages/63/e2/9f66ca6a651a52abfe0d4964ce01439ed34f3f1e119de10ff3a07f403043/matplotlib-3.10.9-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:42fb814efabe95c06c1994d8ab5a8385f43a249e23badd3ba931d4308e5bca20", size = 8304420, upload-time = "2026-04-24T00:14:04.57Z" }, + { url = "https://files.pythonhosted.org/packages/e8/e8/467c03568218792906aa87b5e7bb379b605e056ed0c74fe00c051786d925/matplotlib-3.10.9-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:f76e640a5268850bfda54b5131b1b1941cc685e42c5fa98ed9f2d64038308cba", size = 8197981, upload-time = "2026-04-24T00:14:07.233Z" }, + { url = "https://files.pythonhosted.org/packages/6f/87/afead29192170917537934c6aff4b008c805fff7b1ccea0c79120d96beda/matplotlib-3.10.9-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3fc0364dfbe1d07f6d15c5ebd0c5bf89e126916e5a8667dd4a7a6e84c36653d4", size = 8774002, upload-time = "2026-04-24T00:14:09.816Z" }, +] + +[[package]] +name = "moviepy" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "decorator" }, + { name = "imageio" }, + { name = "imageio-ffmpeg" }, + { name = "numpy" }, + { name = "pillow" }, + { name = "proglog" }, + { name = "python-dotenv" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/de/61/15f9476e270f64c78a834e7459ca045d669f869cec24eed26807b8cd479d/moviepy-2.2.1.tar.gz", hash = "sha256:c80cb56815ece94e5e3e2d361aa40070eeb30a09d23a24c4e684d03e16deacb1", size = 58431438 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/73/7d3b2010baa0b5eb1e4dfa9e4385e89b6716be76f2fa21a6c0fe34b68e5a/moviepy-2.2.1-py3-none-any.whl", hash = "sha256:6b56803fec2ac54b557404126ac1160e65448e03798fa282bd23e8fab3795060", size = 129871 }, +] + +[[package]] +name = "mpmath" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106, upload-time = "2023-03-07T16:47:11.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, +] + +[[package]] +name = "networkx" +version = "3.6.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, +] + +[[package]] +name = "numpy" +version = "2.3.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/76/65/21b3bc86aac7b8f2862db1e808f1ea22b028e30a225a34a5ede9bf8678f2/numpy-2.3.5.tar.gz", hash = "sha256:784db1dcdab56bf0517743e746dfb0f885fc68d948aba86eeec2cba234bdf1c0", size = 20584950, upload-time = "2025-11-16T22:52:42.067Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/77/84dd1d2e34d7e2792a236ba180b5e8fcc1e3e414e761ce0253f63d7f572e/numpy-2.3.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:de5672f4a7b200c15a4127042170a694d4df43c992948f5e1af57f0174beed10", size = 17034641, upload-time = "2025-11-16T22:49:19.336Z" }, + { url = "https://files.pythonhosted.org/packages/2a/ea/25e26fa5837106cde46ae7d0b667e20f69cbbc0efd64cba8221411ab26ae/numpy-2.3.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:acfd89508504a19ed06ef963ad544ec6664518c863436306153e13e94605c218", size = 12528324, upload-time = "2025-11-16T22:49:22.582Z" }, + { url = "https://files.pythonhosted.org/packages/4d/1a/e85f0eea4cf03d6a0228f5c0256b53f2df4bc794706e7df019fc622e47f1/numpy-2.3.5-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:ffe22d2b05504f786c867c8395de703937f934272eb67586817b46188b4ded6d", size = 5356872, upload-time = "2025-11-16T22:49:25.408Z" }, + { url = "https://files.pythonhosted.org/packages/5c/bb/35ef04afd567f4c989c2060cde39211e4ac5357155c1833bcd1166055c61/numpy-2.3.5-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:872a5cf366aec6bb1147336480fef14c9164b154aeb6542327de4970282cd2f5", size = 6893148, upload-time = "2025-11-16T22:49:27.549Z" }, + { url = "https://files.pythonhosted.org/packages/f2/2b/05bbeb06e2dff5eab512dfc678b1cc5ee94d8ac5956a0885c64b6b26252b/numpy-2.3.5-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3095bdb8dd297e5920b010e96134ed91d852d81d490e787beca7e35ae1d89cf7", size = 14557282, upload-time = "2025-11-16T22:49:30.964Z" }, + { url = "https://files.pythonhosted.org/packages/65/fb/2b23769462b34398d9326081fad5655198fcf18966fcb1f1e49db44fbf31/numpy-2.3.5-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8cba086a43d54ca804ce711b2a940b16e452807acebe7852ff327f1ecd49b0d4", size = 16897903, upload-time = "2025-11-16T22:49:34.191Z" }, + { url = "https://files.pythonhosted.org/packages/ac/14/085f4cf05fc3f1e8aa95e85404e984ffca9b2275a5dc2b1aae18a67538b8/numpy-2.3.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6cf9b429b21df6b99f4dee7a1218b8b7ffbbe7df8764dc0bd60ce8a0708fed1e", size = 16341672, upload-time = "2025-11-16T22:49:37.2Z" }, + { url = "https://files.pythonhosted.org/packages/6f/3b/1f73994904142b2aa290449b3bb99772477b5fd94d787093e4f24f5af763/numpy-2.3.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:396084a36abdb603546b119d96528c2f6263921c50df3c8fd7cb28873a237748", size = 18838896, upload-time = "2025-11-16T22:49:39.727Z" }, + { url = "https://files.pythonhosted.org/packages/cd/b9/cf6649b2124f288309ffc353070792caf42ad69047dcc60da85ee85fea58/numpy-2.3.5-cp311-cp311-win32.whl", hash = "sha256:b0c7088a73aef3d687c4deef8452a3ac7c1be4e29ed8bf3b366c8111128ac60c", size = 6563608, upload-time = "2025-11-16T22:49:42.079Z" }, + { url = "https://files.pythonhosted.org/packages/aa/44/9fe81ae1dcc29c531843852e2874080dc441338574ccc4306b39e2ff6e59/numpy-2.3.5-cp311-cp311-win_amd64.whl", hash = "sha256:a414504bef8945eae5f2d7cb7be2d4af77c5d1cb5e20b296c2c25b61dff2900c", size = 13078442, upload-time = "2025-11-16T22:49:43.99Z" }, + { url = "https://files.pythonhosted.org/packages/6d/a7/f99a41553d2da82a20a2f22e93c94f928e4490bb447c9ff3c4ff230581d3/numpy-2.3.5-cp311-cp311-win_arm64.whl", hash = "sha256:0cd00b7b36e35398fa2d16af7b907b65304ef8bb4817a550e06e5012929830fa", size = 10458555, upload-time = "2025-11-16T22:49:47.092Z" }, + { url = "https://files.pythonhosted.org/packages/c6/65/f9dea8e109371ade9c782b4e4756a82edf9d3366bca495d84d79859a0b79/numpy-2.3.5-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:f0963b55cdd70fad460fa4c1341f12f976bb26cb66021a5580329bd498988310", size = 16910689, upload-time = "2025-11-16T22:52:23.247Z" }, + { url = "https://files.pythonhosted.org/packages/00/4f/edb00032a8fb92ec0a679d3830368355da91a69cab6f3e9c21b64d0bb986/numpy-2.3.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:f4255143f5160d0de972d28c8f9665d882b5f61309d8362fdd3e103cf7bf010c", size = 12457053, upload-time = "2025-11-16T22:52:26.367Z" }, + { url = "https://files.pythonhosted.org/packages/16/a4/e8a53b5abd500a63836a29ebe145fc1ab1f2eefe1cfe59276020373ae0aa/numpy-2.3.5-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:a4b9159734b326535f4dd01d947f919c6eefd2d9827466a696c44ced82dfbc18", size = 5285635, upload-time = "2025-11-16T22:52:29.266Z" }, + { url = "https://files.pythonhosted.org/packages/a3/2f/37eeb9014d9c8b3e9c55bc599c68263ca44fdbc12a93e45a21d1d56df737/numpy-2.3.5-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:2feae0d2c91d46e59fcd62784a3a83b3fb677fead592ce51b5a6fbb4f95965ff", size = 6801770, upload-time = "2025-11-16T22:52:31.421Z" }, + { url = "https://files.pythonhosted.org/packages/7d/e4/68d2f474df2cb671b2b6c2986a02e520671295647dad82484cde80ca427b/numpy-2.3.5-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ffac52f28a7849ad7576293c0cb7b9f08304e8f7d738a8cb8a90ec4c55a998eb", size = 14391768, upload-time = "2025-11-16T22:52:33.593Z" }, + { url = "https://files.pythonhosted.org/packages/b8/50/94ccd8a2b141cb50651fddd4f6a48874acb3c91c8f0842b08a6afc4b0b21/numpy-2.3.5-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:63c0e9e7eea69588479ebf4a8a270d5ac22763cc5854e9a7eae952a3908103f7", size = 16729263, upload-time = "2025-11-16T22:52:36.369Z" }, + { url = "https://files.pythonhosted.org/packages/2d/ee/346fa473e666fe14c52fcdd19ec2424157290a032d4c41f98127bfb31ac7/numpy-2.3.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:f16417ec91f12f814b10bafe79ef77e70113a2f5f7018640e7425ff979253425", size = 12967213, upload-time = "2025-11-16T22:52:39.38Z" }, +] + +[[package]] +name = "nvidia-cublas" +version = "13.1.0.3" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/a5/fce49e2ae977e0ccc084e5adafceb4f0ac0c8333cb6863501618a7277f67/nvidia_cublas-13.1.0.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:c86fc7f7ae36d7528288c5d88098edcb7b02c633d262e7ddbb86b0ad91be5df2", size = 542851226, upload-time = "2025-10-09T08:59:04.818Z" }, + { url = "https://files.pythonhosted.org/packages/e7/44/423ac00af4dd95a5aeb27207e2c0d9b7118702149bf4704c3ddb55bb7429/nvidia_cublas-13.1.0.3-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:ee8722c1f0145ab246bccb9e452153b5e0515fd094c3678df50b2a0888b8b171", size = 423133236, upload-time = "2025-10-09T08:59:32.536Z" }, +] + +[[package]] +name = "nvidia-cuda-cupti" +version = "13.0.85" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/2a/80353b103fc20ce05ef51e928daed4b6015db4aaa9162ed0997090fe2250/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_aarch64.whl", hash = "sha256:796bd679890ee55fb14a94629b698b6db54bcfd833d391d5e94017dd9d7d3151", size = 10310827, upload-time = "2025-09-04T08:26:42.012Z" }, + { url = "https://files.pythonhosted.org/packages/33/6d/737d164b4837a9bbd202f5ae3078975f0525a55730fe871d8ed4e3b952b0/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl", hash = "sha256:4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8", size = 10715597, upload-time = "2025-09-04T08:26:51.312Z" }, +] + +[[package]] +name = "nvidia-cuda-nvrtc" +version = "13.0.88" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/68/483a78f5e8f31b08fb1bb671559968c0ca3a065ac7acabfc7cee55214fd6/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575", size = 90215200, upload-time = "2025-09-04T08:28:44.204Z" }, + { url = "https://files.pythonhosted.org/packages/b7/dc/6bb80850e0b7edd6588d560758f17e0550893a1feaf436807d64d2da040f/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d27f20a0ca67a4bb34268a5e951033496c5b74870b868bacd046b1b8e0c3267b", size = 43015449, upload-time = "2025-09-04T08:28:20.239Z" }, +] + +[[package]] +name = "nvidia-cuda-runtime" +version = "13.0.96" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/4f/17d7b9b8e285199c58ce28e31b5c5bbaa4d8271af06a89b6405258245de2/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ef9bcbe90493a2b9d810e43d249adb3d02e98dd30200d86607d8d02687c43f55", size = 2261060, upload-time = "2025-10-09T08:55:15.78Z" }, + { url = "https://files.pythonhosted.org/packages/2e/24/d1558f3b68b1d26e706813b1d10aa1d785e4698c425af8db8edc3dced472/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548", size = 2243632, upload-time = "2025-10-09T08:55:36.117Z" }, +] + +[[package]] +name = "nvidia-cudnn-cu13" +version = "9.19.0.56" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/84/26025437c1e6b61a707442184fa0c03d083b661adf3a3eecfd6d21677740/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:6ed29ffaee1176c612daf442e4dd6cfeb6a0caa43ddcbeb59da94953030b1be4", size = 433781201, upload-time = "2026-02-03T20:40:53.805Z" }, + { url = "https://files.pythonhosted.org/packages/a3/22/0b4b932655d17a6da1b92fa92ab12844b053bb2ac2475e179ba6f043da1e/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:d20e1734305e9d68889a96e3f35094d733ff1f83932ebe462753973e53a572bf", size = 366066321, upload-time = "2026-02-03T20:44:52.837Z" }, +] + +[[package]] +name = "nvidia-cufft" +version = "12.0.0.61" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, + { url = "https://files.pythonhosted.org/packages/a8/2f/7b57e29836ea8714f81e9898409196f47d772d5ddedddf1592eadb8ab743/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c44f692dce8fd5ffd3e3df134b6cdb9c2f72d99cf40b62c32dde45eea9ddad3", size = 214085489, upload-time = "2025-09-04T08:31:56.044Z" }, +] + +[[package]] +name = "nvidia-cufile" +version = "1.15.1.6" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/70/4f193de89a48b71714e74602ee14d04e4019ad36a5a9f20c425776e72cd6/nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08a3ecefae5a01c7f5117351c64f17c7c62efa5fffdbe24fc7d298da19cd0b44", size = 1223672, upload-time = "2025-09-04T08:32:22.779Z" }, + { url = "https://files.pythonhosted.org/packages/ab/73/cc4a14c9813a8a0d509417cf5f4bdaba76e924d58beb9864f5a7baceefbf/nvidia_cufile-1.15.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:bdc0deedc61f548bddf7733bdc216456c2fdb101d020e1ab4b88d232d5e2f6d1", size = 1136992, upload-time = "2025-09-04T08:32:14.119Z" }, +] + +[[package]] +name = "nvidia-curand" +version = "10.4.0.35" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/72/7c2ae24fb6b63a32e6ae5d241cc65263ea18d08802aaae087d9f013335a2/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:133df5a7509c3e292aaa2b477afd0194f06ce4ea24d714d616ff36439cee349a", size = 61962106, upload-time = "2025-08-04T10:21:41.128Z" }, + { url = "https://files.pythonhosted.org/packages/a5/9f/be0a41ca4a4917abf5cb9ae0daff1a6060cc5de950aec0396de9f3b52bc5/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:1aee33a5da6e1db083fe2b90082def8915f30f3248d5896bcec36a579d941bfc", size = 59544258, upload-time = "2025-08-04T10:22:03.992Z" }, +] + +[[package]] +name = "nvidia-cusolver" +version = "12.0.4.66" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas" }, + { name = "nvidia-cusparse" }, + { name = "nvidia-nvjitlink" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, + { url = "https://files.pythonhosted.org/packages/5f/67/cba3777620cdacb99102da4042883709c41c709f4b6323c10781a9c3aa34/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0a759da5dea5c0ea10fd307de75cdeb59e7ea4fcb8add0924859b944babf1112", size = 200941980, upload-time = "2025-09-04T08:33:22.767Z" }, +] + +[[package]] +name = "nvidia-cusparse" +version = "12.6.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, + { url = "https://files.pythonhosted.org/packages/fa/18/623c77619c31d62efd55302939756966f3ecc8d724a14dab2b75f1508850/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b3c89c88d01ee0e477cb7f82ef60a11a4bcd57b6b87c33f789350b59759360b", size = 145942937, upload-time = "2025-09-04T08:33:58.029Z" }, +] + +[[package]] +name = "nvidia-cusparselt-cu13" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/10/8dcd1175260706a2fc92a16a52e306b71d4c1ea0b0cc4a9484183399818a/nvidia_cusparselt_cu13-0.8.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:400c6ed1cf6780fc6efedd64ec9f1345871767e6a1a0a552a1ea0578117ea77c", size = 220791277, upload-time = "2025-08-13T19:22:40.982Z" }, + { url = "https://files.pythonhosted.org/packages/fd/53/43b0d71f4e702fa9733f8b4571fdca50a8813f1e450b656c239beff12315/nvidia_cusparselt_cu13-0.8.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:25e30a8a7323935d4ad0340b95a0b69926eee755767e8e0b1cf8dd85b197d3fd", size = 169884119, upload-time = "2025-08-13T19:23:41.967Z" }, +] + +[[package]] +name = "nvidia-nccl-cu13" +version = "2.28.9" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/55/1920646a2e43ffd4fc958536b276197ed740e9e0c54105b4bb3521591fc7/nvidia_nccl_cu13-2.28.9-py3-none-manylinux_2_18_aarch64.whl", hash = "sha256:01c873ba1626b54caa12272ed228dc5b2781545e0ae8ba3f432a8ef1c6d78643", size = 196561677, upload-time = "2025-11-18T05:49:03.45Z" }, + { url = "https://files.pythonhosted.org/packages/b0/b4/878fefaad5b2bcc6fcf8d474a25e3e3774bc5133e4b58adff4d0bca238bc/nvidia_nccl_cu13-2.28.9-py3-none-manylinux_2_18_x86_64.whl", hash = "sha256:e4553a30f34195f3fa1da02a6da3d6337d28f2003943aa0a3d247bbc25fefc42", size = 196493177, upload-time = "2025-11-18T05:49:17.677Z" }, +] + +[[package]] +name = "nvidia-nvjitlink" +version = "13.0.88" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/7a/123e033aaff487c77107195fa5a2b8686795ca537935a24efae476c41f05/nvidia_nvjitlink-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:13a74f429e23b921c1109976abefacc69835f2f433ebd323d3946e11d804e47b", size = 40713933, upload-time = "2025-09-04T08:35:43.553Z" }, + { url = "https://files.pythonhosted.org/packages/ab/2c/93c5250e64df4f894f1cbb397c6fd71f79813f9fd79d7cd61de3f97b3c2d/nvidia_nvjitlink-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e931536ccc7d467a98ba1d8b89ff7fa7f1fa3b13f2b0069118cd7f47bff07d0c", size = 38768748, upload-time = "2025-09-04T08:35:20.008Z" }, +] + +[[package]] +name = "nvidia-nvshmem-cu13" +version = "3.4.5" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/0f/05cc9c720236dcd2db9c1ab97fff629e96821be2e63103569da0c9b72f19/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dc2a197f38e5d0376ad52cd1a2a3617d3cdc150fd5966f4aee9bcebb1d68fe9", size = 60215947, upload-time = "2025-09-06T00:32:20.022Z" }, + { url = "https://files.pythonhosted.org/packages/3c/35/a9bf80a609e74e3b000fef598933235c908fcefcef9026042b8e6dfde2a9/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:290f0a2ee94c9f3687a02502f3b9299a9f9fe826e6d0287ee18482e78d495b80", size = 60412546, upload-time = "2025-09-06T00:32:41.564Z" }, +] + +[[package]] +name = "nvidia-nvtx" +version = "13.0.85" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/f3/d86c845465a2723ad7e1e5c36dcd75ddb82898b3f53be47ebd429fb2fa5d/nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4936d1d6780fbe68db454f5e72a42ff64d1fd6397df9f363ae786930fd5c1cd4", size = 148047, upload-time = "2025-09-04T08:29:01.761Z" }, + { url = "https://files.pythonhosted.org/packages/a8/64/3708a90d1ebe202ffdeb7185f878a3c84d15c2b2c31858da2ce0583e2def/nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6", size = 148878, upload-time = "2025-09-04T08:28:53.627Z" }, +] + +[[package]] +name = "opencv-python-headless" +version = "4.13.0.92" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/42/2310883be3b8826ac58c3f2787b9358a2d46923d61f88fedf930bc59c60c/opencv_python_headless-4.13.0.92-cp37-abi3-macosx_13_0_arm64.whl", hash = "sha256:1a7d040ac656c11b8c38677cc8cccdc149f98535089dbe5b081e80a4e5903209", size = 46247192, upload-time = "2026-02-05T07:01:35.187Z" }, + { url = "https://files.pythonhosted.org/packages/2d/1e/6f9e38005a6f7f22af785df42a43139d0e20f169eb5787ce8be37ee7fcc9/opencv_python_headless-4.13.0.92-cp37-abi3-macosx_14_0_x86_64.whl", hash = "sha256:3e0a6f0a37994ec6ce5f59e936be21d5d6384a4556f2d2da9c2f9c5dc948394c", size = 32568914, upload-time = "2026-02-05T07:01:51.989Z" }, + { url = "https://files.pythonhosted.org/packages/21/76/9417a6aef9def70e467a5bf560579f816148a4c658b7d525581b356eda9e/opencv_python_headless-4.13.0.92-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5c8cfc8e87ed452b5cecb9419473ee5560a989859fe1d10d1ce11ae87b09a2cb", size = 33703709, upload-time = "2026-02-05T10:24:46.469Z" }, + { url = "https://files.pythonhosted.org/packages/92/ce/bd17ff5772938267fd49716e94ca24f616ff4cb1ff4c6be13085108037be/opencv_python_headless-4.13.0.92-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0525a3d2c0b46c611e2130b5fdebc94cf404845d8fa64d2f3a3b679572a5bd22", size = 56016764, upload-time = "2026-02-05T10:26:48.904Z" }, + { url = "https://files.pythonhosted.org/packages/8f/b4/b7bcbf7c874665825a8c8e1097e93ea25d1f1d210a3e20d4451d01da30aa/opencv_python_headless-4.13.0.92-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:eb60e36b237b1ebd40a912da5384b348df8ed534f6f644d8e0b4f103e272ba7d", size = 35010236, upload-time = "2026-02-05T10:28:11.031Z" }, + { url = "https://files.pythonhosted.org/packages/4b/33/b5db29a6c00eb8f50708110d8d453747ca125c8b805bc437b289dbdcc057/opencv_python_headless-4.13.0.92-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:0bd48544f77c68b2941392fcdf9bcd2b9cdf00e98cb8c29b2455d194763cf99e", size = 60391106, upload-time = "2026-02-05T10:30:14.236Z" }, + { url = "https://files.pythonhosted.org/packages/fb/c3/52cfea47cd33e53e8c0fbd6e7c800b457245c1fda7d61660b4ffe9596a7f/opencv_python_headless-4.13.0.92-cp37-abi3-win32.whl", hash = "sha256:a7cf08e5b191f4ebb530791acc0825a7986e0d0dee2a3c491184bd8599848a4b", size = 30812232, upload-time = "2026-02-05T07:02:29.594Z" }, + { url = "https://files.pythonhosted.org/packages/4a/90/b338326131ccb2aaa3c2c85d00f41822c0050139a4bfe723cfd95455bd2d/opencv_python_headless-4.13.0.92-cp37-abi3-win_amd64.whl", hash = "sha256:77a82fe35ddcec0f62c15f2ba8a12ecc2ed4207c17b0902c7a3151ae29f37fb6", size = 40070414, upload-time = "2026-02-05T07:02:26.448Z" }, +] + +[[package]] +name = "optree" +version = "0.19.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/44/63/92328a17ab7836562fe0129e605f685a88db35ce98427c34ff48ee4ec157/optree-0.19.1.tar.gz", hash = "sha256:4497d1c9197b8c6842e511368163d318ce536521ebdcff8bebb7551dcdfac532", size = 177531 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/f2/4671a78193f96e86c1343fe04324091e163973d0058b292c10bc3387bb70/optree-0.19.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a496b864fe1fe0b5ea23d1ee3d1ef958d910704661808db2b2d2e16a0cfac96c", size = 414314 }, + { url = "https://files.pythonhosted.org/packages/d6/93/7decea24656f416d61fa57b7113b1fbdbc042b7ab421399a84e1755676a1/optree-0.19.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c1667e502e0eda9477925fb17c2ad879b199a2283ac98f18e6453692819b7811", size = 385006 }, + { url = "https://files.pythonhosted.org/packages/af/2e/9d1bd2527481681c4399beeeabba11dca36b16ec814579f2e8cc6bc2af96/optree-0.19.1-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:42e367a9d81e57c31a23247094727987a2f64b708901233a42a24d44d24e93f6", size = 406124 }, + { url = "https://files.pythonhosted.org/packages/df/29/cdb40de6307809fa8e9452e4f9a65881a3140d01d9d589a07e9d054d8e1c/optree-0.19.1-cp311-cp311-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:96fad6c7b3a6fde3a0c8655fd003359cd247f8400749217502591a5ffc328699", size = 466772 }, + { url = "https://files.pythonhosted.org/packages/cb/15/4645e1816e815a1306bbb7e3e2e6ba124f6dc325f8088a2db69301219a0c/optree-0.19.1-cp311-cp311-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f3a900df0ffb9b8259961b337289754531a7e0a5de2f681e9c80866b6a7cb74e", size = 466203 }, + { url = "https://files.pythonhosted.org/packages/e6/d2/5758c76bdd7034b721d84c7f0fd911f3b39dcb489eeb27f674aaae8a5f5c/optree-0.19.1-cp311-cp311-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:27c8dc0f89ade9233aa7ed25ce15991da188e6950eb17cc0c313fc1f327c5b0b", size = 465030 }, + { url = "https://files.pythonhosted.org/packages/09/b9/f668bc51129c0fec7728ae8b43180417fe1c1fe99f71d302739f6cc50944/optree-0.19.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38f2e503fad50aff58cade85db448002d4adc72f4b3b50dcc7f3ef4bcd3b0173", size = 447141 }, + { url = "https://files.pythonhosted.org/packages/a4/08/a7b8862e4465bf250c3ccc78db4d10b9a2cf90ce4db3681cbdf7eb076fb7/optree-0.19.1-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:5dc35cb31540ab6ed9850b0f8865ccd400994ebd51fcf0c156cc772073f43c04", size = 410016 }, + { url = "https://files.pythonhosted.org/packages/fb/04/04b71a34cf5e663a1df029acceb5efc8a96c8dc4b0b6af6e98486638e913/optree-0.19.1-cp311-cp311-win32.whl", hash = "sha256:d32b1261be71211f77837e839e43a3e3e8fc57707091d2454d0a88590fb6abe8", size = 311810 }, + { url = "https://files.pythonhosted.org/packages/22/64/3cc7b08cb1c0f1949895f9490217ca8db6ced7f3bf75c65a5bf31c07bf1e/optree-0.19.1-cp311-cp311-win_amd64.whl", hash = "sha256:cd28a527bb363a1d7d28e8b2fb62816ace6743418bb86e9c5f27ea6877dcdf6c", size = 337620 }, + { url = "https://files.pythonhosted.org/packages/6c/14/85f4b05765287658529f09ede10461224161dcf0e29e6fce1ae488451cfe/optree-0.19.1-cp311-cp311-win_arm64.whl", hash = "sha256:7853b58aa084e882ea078f390936bd92e46972eb8f9b5e654360b6480ca7283b", size = 349337 }, + { url = "https://files.pythonhosted.org/packages/2b/d4/ffeedc86f8b91e5c17994f38bd1f7aa2e20f9b70a6d3ae906af16414626c/optree-0.19.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:3f4f1c276fa06227cdaf58349d22a3231b3dd3d47de1f90a86222ebf831fc397", size = 417543 }, + { url = "https://files.pythonhosted.org/packages/52/0b/80fb1b289940e34858cb89f05bc7ce23d6d1272886c2f78bc7e3ab1a306b/optree-0.19.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:77d93eafbd0046c7350bc592ab8e3814abbd39a6d716b5b1e5d652cc486f445c", size = 390184 }, + { url = "https://files.pythonhosted.org/packages/fc/67/f31784a7a2dcc0c1f84b691afc552ea5b26db5f56657692a12954a828db4/optree-0.19.1-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3507ae5db5827eef3da42d04c5a41df649cedc2e42d5d39dc0f869d36915a00b", size = 409025 }, + { url = "https://files.pythonhosted.org/packages/3e/a5/647b93eb16244cc7f6dfccc025ac495245e306ff4cb8f9ad15718219141a/optree-0.19.1-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:732c4581fb666869b8b391ec4ca13d2729795f9abe72b5aec2e582bcbea1975d", size = 449514 }, + { url = "https://files.pythonhosted.org/packages/c7/8e/d251c9338771ef0f9db8e538bd77810100c495734b57494464c7e223f0d0/optree-0.19.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:e12ee3776a16f6feaa8263b92469ad546b870af71d50602745855d8449219221", size = 341586 }, +] + +[[package]] +name = "optree" +version = "0.19.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/44/63/92328a17ab7836562fe0129e605f685a88db35ce98427c34ff48ee4ec157/optree-0.19.1.tar.gz", hash = "sha256:4497d1c9197b8c6842e511368163d318ce536521ebdcff8bebb7551dcdfac532", size = 177531 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/f2/4671a78193f96e86c1343fe04324091e163973d0058b292c10bc3387bb70/optree-0.19.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a496b864fe1fe0b5ea23d1ee3d1ef958d910704661808db2b2d2e16a0cfac96c", size = 414314 }, + { url = "https://files.pythonhosted.org/packages/d6/93/7decea24656f416d61fa57b7113b1fbdbc042b7ab421399a84e1755676a1/optree-0.19.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c1667e502e0eda9477925fb17c2ad879b199a2283ac98f18e6453692819b7811", size = 385006 }, + { url = "https://files.pythonhosted.org/packages/af/2e/9d1bd2527481681c4399beeeabba11dca36b16ec814579f2e8cc6bc2af96/optree-0.19.1-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:42e367a9d81e57c31a23247094727987a2f64b708901233a42a24d44d24e93f6", size = 406124 }, + { url = "https://files.pythonhosted.org/packages/df/29/cdb40de6307809fa8e9452e4f9a65881a3140d01d9d589a07e9d054d8e1c/optree-0.19.1-cp311-cp311-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:96fad6c7b3a6fde3a0c8655fd003359cd247f8400749217502591a5ffc328699", size = 466772 }, + { url = "https://files.pythonhosted.org/packages/cb/15/4645e1816e815a1306bbb7e3e2e6ba124f6dc325f8088a2db69301219a0c/optree-0.19.1-cp311-cp311-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f3a900df0ffb9b8259961b337289754531a7e0a5de2f681e9c80866b6a7cb74e", size = 466203 }, + { url = "https://files.pythonhosted.org/packages/e6/d2/5758c76bdd7034b721d84c7f0fd911f3b39dcb489eeb27f674aaae8a5f5c/optree-0.19.1-cp311-cp311-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:27c8dc0f89ade9233aa7ed25ce15991da188e6950eb17cc0c313fc1f327c5b0b", size = 465030 }, + { url = "https://files.pythonhosted.org/packages/09/b9/f668bc51129c0fec7728ae8b43180417fe1c1fe99f71d302739f6cc50944/optree-0.19.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38f2e503fad50aff58cade85db448002d4adc72f4b3b50dcc7f3ef4bcd3b0173", size = 447141 }, + { url = "https://files.pythonhosted.org/packages/a4/08/a7b8862e4465bf250c3ccc78db4d10b9a2cf90ce4db3681cbdf7eb076fb7/optree-0.19.1-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:5dc35cb31540ab6ed9850b0f8865ccd400994ebd51fcf0c156cc772073f43c04", size = 410016 }, + { url = "https://files.pythonhosted.org/packages/fb/04/04b71a34cf5e663a1df029acceb5efc8a96c8dc4b0b6af6e98486638e913/optree-0.19.1-cp311-cp311-win32.whl", hash = "sha256:d32b1261be71211f77837e839e43a3e3e8fc57707091d2454d0a88590fb6abe8", size = 311810 }, + { url = "https://files.pythonhosted.org/packages/22/64/3cc7b08cb1c0f1949895f9490217ca8db6ced7f3bf75c65a5bf31c07bf1e/optree-0.19.1-cp311-cp311-win_amd64.whl", hash = "sha256:cd28a527bb363a1d7d28e8b2fb62816ace6743418bb86e9c5f27ea6877dcdf6c", size = 337620 }, + { url = "https://files.pythonhosted.org/packages/6c/14/85f4b05765287658529f09ede10461224161dcf0e29e6fce1ae488451cfe/optree-0.19.1-cp311-cp311-win_arm64.whl", hash = "sha256:7853b58aa084e882ea078f390936bd92e46972eb8f9b5e654360b6480ca7283b", size = 349337 }, + { url = "https://files.pythonhosted.org/packages/2b/d4/ffeedc86f8b91e5c17994f38bd1f7aa2e20f9b70a6d3ae906af16414626c/optree-0.19.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:3f4f1c276fa06227cdaf58349d22a3231b3dd3d47de1f90a86222ebf831fc397", size = 417543 }, + { url = "https://files.pythonhosted.org/packages/52/0b/80fb1b289940e34858cb89f05bc7ce23d6d1272886c2f78bc7e3ab1a306b/optree-0.19.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:77d93eafbd0046c7350bc592ab8e3814abbd39a6d716b5b1e5d652cc486f445c", size = 390184 }, + { url = "https://files.pythonhosted.org/packages/fc/67/f31784a7a2dcc0c1f84b691afc552ea5b26db5f56657692a12954a828db4/optree-0.19.1-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3507ae5db5827eef3da42d04c5a41df649cedc2e42d5d39dc0f869d36915a00b", size = 409025 }, + { url = "https://files.pythonhosted.org/packages/3e/a5/647b93eb16244cc7f6dfccc025ac495245e306ff4cb8f9ad15718219141a/optree-0.19.1-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:732c4581fb666869b8b391ec4ca13d2729795f9abe72b5aec2e582bcbea1975d", size = 449514 }, + { url = "https://files.pythonhosted.org/packages/c7/8e/d251c9338771ef0f9db8e538bd77810100c495734b57494464c7e223f0d0/optree-0.19.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:e12ee3776a16f6feaa8263b92469ad546b870af71d50602745855d8449219221", size = 341586 }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pillow" +version = "12.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/21/c2bcdd5906101a30244eaffc1b6e6ce71a31bd0742a01eb89e660ebfac2d/pillow-12.2.0.tar.gz", hash = "sha256:a830b1a40919539d07806aa58e1b114df53ddd43213d9c8b75847eee6c0182b5", size = 46987819, upload-time = "2026-04-01T14:46:17.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/e1/748f5663efe6edcfc4e74b2b93edfb9b8b99b67f21a854c3ae416500a2d9/pillow-12.2.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:8be29e59487a79f173507c30ddf57e733a357f67881430449bb32614075a40ab", size = 5354347, upload-time = "2026-04-01T14:42:44.255Z" }, + { url = "https://files.pythonhosted.org/packages/47/a1/d5ff69e747374c33a3b53b9f98cca7889fce1fd03d79cdc4e1bccc6c5a87/pillow-12.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:71cde9a1e1551df7d34a25462fc60325e8a11a82cc2e2f54578e5e9a1e153d65", size = 4695873, upload-time = "2026-04-01T14:42:46.452Z" }, + { url = "https://files.pythonhosted.org/packages/df/21/e3fbdf54408a973c7f7f89a23b2cb97a7ef30c61ab4142af31eee6aebc88/pillow-12.2.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f490f9368b6fc026f021db16d7ec2fbf7d89e2edb42e8ec09d2c60505f5729c7", size = 6280168, upload-time = "2026-04-01T14:42:49.228Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f1/00b7278c7dd52b17ad4329153748f87b6756ec195ff786c2bdf12518337d/pillow-12.2.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8bd7903a5f2a4545f6fd5935c90058b89d30045568985a71c79f5fd6edf9b91e", size = 8088188, upload-time = "2026-04-01T14:42:51.735Z" }, + { url = "https://files.pythonhosted.org/packages/ad/cf/220a5994ef1b10e70e85748b75649d77d506499352be135a4989c957b701/pillow-12.2.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3997232e10d2920a68d25191392e3a4487d8183039e1c74c2297f00ed1c50705", size = 6394401, upload-time = "2026-04-01T14:42:54.343Z" }, + { url = "https://files.pythonhosted.org/packages/e9/bd/e51a61b1054f09437acfbc2ff9106c30d1eb76bc1453d428399946781253/pillow-12.2.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e74473c875d78b8e9d5da2a70f7099549f9eb37ded4e2f6a463e60125bccd176", size = 7079655, upload-time = "2026-04-01T14:42:56.954Z" }, + { url = "https://files.pythonhosted.org/packages/6b/3d/45132c57d5fb4b5744567c3817026480ac7fc3ce5d4c47902bc0e7f6f853/pillow-12.2.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:56a3f9c60a13133a98ecff6197af34d7824de9b7b38c3654861a725c970c197b", size = 6503105, upload-time = "2026-04-01T14:42:59.847Z" }, + { url = "https://files.pythonhosted.org/packages/7d/2e/9df2fc1e82097b1df3dce58dc43286aa01068e918c07574711fcc53e6fb4/pillow-12.2.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:90e6f81de50ad6b534cab6e5aef77ff6e37722b2f5d908686f4a5c9eba17a909", size = 7203402, upload-time = "2026-04-01T14:43:02.664Z" }, + { url = "https://files.pythonhosted.org/packages/bd/2e/2941e42858ebb67e50ae741473de81c2984e6eff7b397017623c676e2e8d/pillow-12.2.0-cp311-cp311-win32.whl", hash = "sha256:8c984051042858021a54926eb597d6ee3012393ce9c181814115df4c60b9a808", size = 6378149, upload-time = "2026-04-01T14:43:05.274Z" }, + { url = "https://files.pythonhosted.org/packages/69/42/836b6f3cd7f3e5fa10a1f1a5420447c17966044c8fbf589cc0452d5502db/pillow-12.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:6e6b2a0c538fc200b38ff9eb6628228b77908c319a005815f2dde585a0664b60", size = 7082626, upload-time = "2026-04-01T14:43:08.557Z" }, + { url = "https://files.pythonhosted.org/packages/c2/88/549194b5d6f1f494b485e493edc6693c0a16f4ada488e5bd974ed1f42fad/pillow-12.2.0-cp311-cp311-win_arm64.whl", hash = "sha256:9a8a34cc89c67a65ea7437ce257cea81a9dad65b29805f3ecee8c8fe8ff25ffe", size = 2463531, upload-time = "2026-04-01T14:43:10.743Z" }, + { url = "https://files.pythonhosted.org/packages/4e/b7/2437044fb910f499610356d1352e3423753c98e34f915252aafecc64889f/pillow-12.2.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0538bd5e05efec03ae613fd89c4ce0368ecd2ba239cc25b9f9be7ed426b0af1f", size = 5273969, upload-time = "2026-04-01T14:45:55.538Z" }, + { url = "https://files.pythonhosted.org/packages/f6/f4/8316e31de11b780f4ac08ef3654a75555e624a98db1056ecb2122d008d5a/pillow-12.2.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:394167b21da716608eac917c60aa9b969421b5dcbbe02ae7f013e7b85811c69d", size = 4659674, upload-time = "2026-04-01T14:45:58.093Z" }, + { url = "https://files.pythonhosted.org/packages/d4/37/664fca7201f8bb2aa1d20e2c3d5564a62e6ae5111741966c8319ca802361/pillow-12.2.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5d04bfa02cc2d23b497d1e90a0f927070043f6cbf303e738300532379a4b4e0f", size = 5288479, upload-time = "2026-04-01T14:46:01.141Z" }, + { url = "https://files.pythonhosted.org/packages/49/62/5b0ed78fce87346be7a5cfcfaaad91f6a1f98c26f86bdbafa2066c647ef6/pillow-12.2.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0c838a5125cee37e68edec915651521191cef1e6aa336b855f495766e77a366e", size = 7032230, upload-time = "2026-04-01T14:46:03.874Z" }, + { url = "https://files.pythonhosted.org/packages/c3/28/ec0fc38107fc32536908034e990c47914c57cd7c5a3ece4d8d8f7ffd7e27/pillow-12.2.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a6c9fa44005fa37a91ebfc95d081e8079757d2e904b27103f4f5fa6f0bf78c0", size = 5355404, upload-time = "2026-04-01T14:46:06.33Z" }, + { url = "https://files.pythonhosted.org/packages/5e/8b/51b0eddcfa2180d60e41f06bd6d0a62202b20b59c68f5a132e615b75aecf/pillow-12.2.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:25373b66e0dd5905ed63fa3cae13c82fbddf3079f2c8bf15c6fb6a35586324c1", size = 6002215, upload-time = "2026-04-01T14:46:08.83Z" }, + { url = "https://files.pythonhosted.org/packages/bc/60/5382c03e1970de634027cee8e1b7d39776b778b81812aaf45b694dfe9e28/pillow-12.2.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:bfa9c230d2fe991bed5318a5f119bd6780cda2915cca595393649fc118ab895e", size = 7080946, upload-time = "2026-04-01T14:46:11.734Z" }, +] + +[[package]] +name = "platformdirs" +version = "4.9.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9f/4a/0883b8e3802965322523f0b200ecf33d31f10991d0401162f4b23c698b42/platformdirs-4.9.6.tar.gz", hash = "sha256:3bfa75b0ad0db84096ae777218481852c0ebc6c727b3168c1b9e0118e458cf0a", size = 29400, upload-time = "2026-04-09T00:04:10.812Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/75/a6/a0a304dc33b49145b21f4808d763822111e67d1c3a32b524a1baf947b6e1/platformdirs-4.9.6-py3-none-any.whl", hash = "sha256:e61adb1d5e5cb3441b4b7710bea7e4c12250ca49439228cc1021c00dcfac0917", size = 21348, upload-time = "2026-04-09T00:04:09.463Z" }, +] + +[[package]] +name = "proglog" +version = "0.1.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c2/af/c108866c452eda1132f3d6b3cb6be2ae8430c97e9309f38ca9dbd430af37/proglog-0.1.12.tar.gz", hash = "sha256:361ee074721c277b89b75c061336cb8c5f287c92b043efa562ccf7866cda931c", size = 8794 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/1b/f7ea6cde25621cd9236541c66ff018f4268012a534ec31032bcb187dc5e7/proglog-0.1.12-py3-none-any.whl", hash = "sha256:ccaafce51e80a81c65dc907a460c07ccb8ec1f78dc660cfd8f9ec3a22f01b84c", size = 6337 }, +] + +[[package]] +name = "protobuf" +version = "7.34.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6b/6b/a0e95cad1ad7cc3f2c6821fcab91671bd5b78bd42afb357bb4765f29bc41/protobuf-7.34.1.tar.gz", hash = "sha256:9ce42245e704cc5027be797c1db1eb93184d44d1cdd71811fb2d9b25ad541280", size = 454708, upload-time = "2026-03-20T17:34:47.036Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/11/3325d41e6ee15bf1125654301211247b042563bcc898784351252549a8ad/protobuf-7.34.1-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:d8b2cc79c4d8f62b293ad9b11ec3aebce9af481fa73e64556969f7345ebf9fc7", size = 429247, upload-time = "2026-03-20T17:34:37.024Z" }, + { url = "https://files.pythonhosted.org/packages/eb/9d/aa69df2724ff63efa6f72307b483ce0827f4347cc6d6df24b59e26659fef/protobuf-7.34.1-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:5185e0e948d07abe94bb76ec9b8416b604cfe5da6f871d67aad30cbf24c3110b", size = 325753, upload-time = "2026-03-20T17:34:38.751Z" }, + { url = "https://files.pythonhosted.org/packages/92/e8/d174c91fd48e50101943f042b09af9029064810b734e4160bbe282fa1caa/protobuf-7.34.1-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:403b093a6e28a960372b44e5eb081775c9b056e816a8029c61231743d63f881a", size = 340198, upload-time = "2026-03-20T17:34:39.871Z" }, + { url = "https://files.pythonhosted.org/packages/53/1b/3b431694a4dc6d37b9f653f0c64b0a0d9ec074ee810710c0c3da21d67ba7/protobuf-7.34.1-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:8ff40ce8cd688f7265326b38d5a1bed9bfdf5e6723d49961432f83e21d5713e4", size = 324267, upload-time = "2026-03-20T17:34:41.1Z" }, + { url = "https://files.pythonhosted.org/packages/85/29/64de04a0ac142fb685fd09999bc3d337943fb386f3a0ec57f92fd8203f97/protobuf-7.34.1-cp310-abi3-win32.whl", hash = "sha256:34b84ce27680df7cca9f231043ada0daa55d0c44a2ddfaa58ec1d0d89d8bf60a", size = 426628, upload-time = "2026-03-20T17:34:42.536Z" }, + { url = "https://files.pythonhosted.org/packages/4d/87/cb5e585192a22b8bd457df5a2c16a75ea0db9674c3a0a39fc9347d84e075/protobuf-7.34.1-cp310-abi3-win_amd64.whl", hash = "sha256:e97b55646e6ce5cbb0954a8c28cd39a5869b59090dfaa7df4598a7fba869468c", size = 437901, upload-time = "2026-03-20T17:34:44.112Z" }, + { url = "https://files.pythonhosted.org/packages/88/95/608f665226bca68b736b79e457fded9a2a38c4f4379a4a7614303d9db3bc/protobuf-7.34.1-py3-none-any.whl", hash = "sha256:bb3812cd53aefea2b028ef42bd780f5b96407247f20c6ef7c679807e9d188f11", size = 170715, upload-time = "2026-03-20T17:34:45.384Z" }, +] + +[[package]] +name = "pydantic" +version = "2.13.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/fa/6d7708d2cfc1a832acb6aeb0cd16e801902df8a0f583bb3b4b527fde022e/pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594", size = 2111872, upload-time = "2026-05-06T13:40:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6f/aa064a3e74b5745afbdf250594f38e7ead05e2d651bcb35994b9417a0d4d/pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c", size = 1948255, upload-time = "2026-05-06T13:39:12.574Z" }, + { url = "https://files.pythonhosted.org/packages/43/3a/41114a9f7569b84b4d84e7a018c57c56347dac30c0d4a872946ec4e36c46/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826", size = 1972827, upload-time = "2026-05-06T13:38:19.841Z" }, + { url = "https://files.pythonhosted.org/packages/ef/25/1ab42e8048fe551934d9884e8d64daa7e990ad386f310a15981aeb6a5b08/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04", size = 2041051, upload-time = "2026-05-06T13:38:10.447Z" }, + { url = "https://files.pythonhosted.org/packages/94/c2/1a934597ddf08da410385b3b7aae91956a5a76c635effef456074fad7e88/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e", size = 2221314, upload-time = "2026-05-06T13:40:13.089Z" }, + { url = "https://files.pythonhosted.org/packages/02/6d/9e8ad178c9c4df27ad3c8f25d1fe2a7ab0d2ba0559fad4aee5d3d1f16771/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3", size = 2285146, upload-time = "2026-05-06T13:38:59.224Z" }, + { url = "https://files.pythonhosted.org/packages/80/50/540cd3aeefc041beb111125c4bff779831a2111fc6b15a9138cda277d32c/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4", size = 2089685, upload-time = "2026-05-06T13:38:17.762Z" }, + { url = "https://files.pythonhosted.org/packages/6b/a4/b440ad35f05f6a38f89fa0f149accb3f0e02be94ca5e15f3c449a61b4bc9/pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398", size = 2115420, upload-time = "2026-05-06T13:37:58.195Z" }, + { url = "https://files.pythonhosted.org/packages/99/61/de4f55db8dfd57bfdfa9a12ec90fe1b57c4f41062f7ca86f08586b3e0ac0/pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3", size = 2165122, upload-time = "2026-05-06T13:37:01.167Z" }, + { url = "https://files.pythonhosted.org/packages/f7/52/7c529d7bdb2d1068bd52f51fe32572c8301f9a4febf1948f10639f1436f5/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848", size = 2182573, upload-time = "2026-05-06T13:38:45.04Z" }, + { url = "https://files.pythonhosted.org/packages/37/b3/7c40325848ba78247f2812dcf9c7274e38cd801820ca6dd9fe63bcfb0eb4/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3", size = 2317139, upload-time = "2026-05-06T13:37:15.539Z" }, + { url = "https://files.pythonhosted.org/packages/d9/37/f913f81a657c865b75da6c0dbed79876073c2a43b5bd9edbe8da785e4d49/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109", size = 2360433, upload-time = "2026-05-06T13:37:30.099Z" }, + { url = "https://files.pythonhosted.org/packages/c4/67/6acaa1be2567f9256b056d8477158cac7240813956ce86e49deae8e173b4/pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda", size = 1985513, upload-time = "2026-05-06T13:38:15.669Z" }, + { url = "https://files.pythonhosted.org/packages/aa/e6/c505f83dfeda9a2e5c995cfd872949e4d05e12f7feb3dca72f633daefa94/pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33", size = 2071114, upload-time = "2026-05-06T13:40:35.416Z" }, + { url = "https://files.pythonhosted.org/packages/0f/da/7a263a96d965d9d0df5e8de8a475f33495451117035b09acb110288c381f/pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d", size = 2044298, upload-time = "2026-05-06T13:38:29.754Z" }, + { url = "https://files.pythonhosted.org/packages/ee/a4/73995fd4ebbb46ba0ee51e6fa049b8f02c40daebb762208feda8a6b7894d/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c", size = 2111589, upload-time = "2026-05-06T13:37:10.817Z" }, + { url = "https://files.pythonhosted.org/packages/fb/7f/f37d3a5e8bfcc2e403f5c57a730f2d815693fb42119e8ea48b3789335af1/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b", size = 1944552, upload-time = "2026-05-06T13:36:56.717Z" }, + { url = "https://files.pythonhosted.org/packages/15/3c/d7eb777b3ff43e8433a4efb39a17aa8fd98a4ee8561a24a67ef5db07b2d6/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b", size = 1982984, upload-time = "2026-05-06T13:39:06.207Z" }, + { url = "https://files.pythonhosted.org/packages/63/87/70b9f40170a81afd55ca26c9b2acb25c20d64bcfbf888fafecb3ba077d4c/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea", size = 2138417, upload-time = "2026-05-06T13:39:45.476Z" }, + { url = "https://files.pythonhosted.org/packages/11/cb/428de0385b6c8d44b716feba566abfacfbd23ee3c4439faa789a1456242f/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0", size = 2112782, upload-time = "2026-05-06T13:37:04.016Z" }, + { url = "https://files.pythonhosted.org/packages/0b/b5/6a17bdadd0fc1f170adfd05a20d37c832f52b117b4d9131da1f41bb097ce/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7", size = 1952146, upload-time = "2026-05-06T13:39:43.092Z" }, + { url = "https://files.pythonhosted.org/packages/2a/dc/03734d80e362cd43ef65428e9de77c730ce7f2f11c60d2b1e1b39f0fbf99/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2", size = 2134492, upload-time = "2026-05-06T13:36:58.124Z" }, + { url = "https://files.pythonhosted.org/packages/de/df/5e5ffc085ed07cc22d298134d3d911c63e91f6a0eb91fe646750a3209910/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9", size = 2156604, upload-time = "2026-05-06T13:37:49.88Z" }, + { url = "https://files.pythonhosted.org/packages/81/44/6e112a4253e56f5705467cbab7ab5e91ee7398ba3d56d358635958893d3e/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf", size = 2183828, upload-time = "2026-05-06T13:37:43.053Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ad/5565071e937d8e752842ac241463944c9eb14c87e2d269f2658a5bd05e98/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30", size = 2310000, upload-time = "2026-05-06T13:37:56.694Z" }, + { url = "https://files.pythonhosted.org/packages/4f/c3/66883a5cec183e7fba4d024b4cbbe61851a63750ef606b0afecc46d1f2bf/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc", size = 2361286, upload-time = "2026-05-06T13:40:05.667Z" }, + { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, +] + +[[package]] +name = "pygame" +version = "2.6.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/cc/08bba60f00541f62aaa252ce0cfbd60aebd04616c0b9574f755b583e45ae/pygame-2.6.1.tar.gz", hash = "sha256:56fb02ead529cee00d415c3e007f75e0780c655909aaa8e8bf616ee09c9feb1f", size = 14808125, upload-time = "2024-09-29T13:41:34.698Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/ca/8f367cb9fe734c4f6f6400e045593beea2635cd736158f9fabf58ee14e3c/pygame-2.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:20349195326a5e82a16e351ed93465a7845a7e2a9af55b7bc1b2110ea3e344e1", size = 13113753, upload-time = "2024-09-29T14:26:13.751Z" }, + { url = "https://files.pythonhosted.org/packages/83/47/6edf2f890139616b3219be9cfcc8f0cb8f42eb15efd59597927e390538cb/pygame-2.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f3935459109da4bb0b3901da9904f0a3e52028a3332a355d298b1673a334cf21", size = 12378146, upload-time = "2024-09-29T14:26:22.456Z" }, + { url = "https://files.pythonhosted.org/packages/00/9e/0d8aa8cf93db2d2ee38ebaf1c7b61d0df36ded27eb726221719c150c673d/pygame-2.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c31dbdb5d0217f32764797d21c2752e258e5fb7e895326538d82b5f75a0cd856", size = 13611760, upload-time = "2024-09-29T11:10:47.317Z" }, + { url = "https://files.pythonhosted.org/packages/d7/9e/d06adaa5cc65876bcd7a24f59f67e07f7e4194e6298130024ed3fb22c456/pygame-2.6.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:173badf82fa198e6888017bea40f511cb28e69ecdd5a72b214e81e4dcd66c3b1", size = 14298054, upload-time = "2024-09-29T11:39:53.891Z" }, + { url = "https://files.pythonhosted.org/packages/7a/a1/9ae2852ebd3a7cc7d9ae7ff7919ab983e4a5c1b7a14e840732f23b2b48f6/pygame-2.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce8cc108b92de9b149b344ad2e25eedbe773af0dc41dfb24d1f07f679b558c60", size = 13977107, upload-time = "2024-09-29T11:39:56.831Z" }, + { url = "https://files.pythonhosted.org/packages/31/df/6788fd2e9a864d0496a77670e44a7c012184b7a5382866ab0e60c55c0f28/pygame-2.6.1-cp311-cp311-win32.whl", hash = "sha256:811e7b925146d8149d79193652cbb83e0eca0aae66476b1cb310f0f4226b8b5c", size = 10250863, upload-time = "2024-09-29T11:44:48.199Z" }, + { url = "https://files.pythonhosted.org/packages/d2/55/ca3eb851aeef4f6f2e98a360c201f0d00bd1ba2eb98e2c7850d80aabc526/pygame-2.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:91476902426facd4bb0dad4dc3b2573bc82c95c71b135e0daaea072ed528d299", size = 10622016, upload-time = "2024-09-29T12:17:01.545Z" }, +] + +[[package]] +name = "pyparsing" +version = "3.3.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/91/9c6ee907786a473bf81c5f53cf703ba0957b23ab84c264080fb5a450416f/pyparsing-3.3.2.tar.gz", hash = "sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc", size = 6851574, upload-time = "2026-01-21T03:57:59.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101 }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, +] + +[[package]] +name = "reinforcement-learning" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "ale-py" }, + { name = "envpool" }, + { name = "gymnasium", extra = ["atari"] }, + { name = "matplotlib" }, + { name = "moviepy" }, + { name = "numpy" }, + { name = "opencv-python-headless" }, + { name = "pygame" }, + { name = "torch" }, + { name = "torchvision" }, + { name = "wandb" }, +] + +[package.metadata] +requires-dist = [ + { name = "ale-py", specifier = ">=0.11,<0.12" }, + { name = "envpool", specifier = ">=1.2.5" }, + { name = "gymnasium", extras = ["atari"], specifier = ">=1.2,<1.3" }, + { name = "matplotlib", specifier = ">=3.10,<3.11" }, + { name = "moviepy", specifier = ">=2.2.1" }, + { name = "numpy", specifier = ">=2.3,<2.4" }, + { name = "opencv-python-headless", specifier = ">=4.13,<4.14" }, + { name = "pygame", specifier = ">=2.6,<3" }, + { name = "torch", specifier = ">=2.11,<2.12" }, + { name = "torchvision", specifier = ">=0.26,<0.27" }, + { name = "wandb", specifier = ">=0.27.0" }, +] + +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + +[[package]] +name = "sentry-sdk" +version = "2.60.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/54/a2/2e6c090db384cc515069f4f85542bd5baf6786852073020ea73d4a76d3ea/sentry_sdk-2.60.0.tar.gz", hash = "sha256:0bd25e54e78ca02d0be512529fa644bbbf9e8470d7b26371294012d4ca93c978", size = 452946, upload-time = "2026-05-13T13:34:52.516Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/41/f2b800b7f12a05dd48c2a6280d4dd812d1425fc66ed3fe3fd99420c41d1a/sentry_sdk-2.60.0-py3-none-any.whl", hash = "sha256:28a536c03291c8bcb363cf35c611b32738ec118ff64d8d6383b096448ac4c803", size = 475616, upload-time = "2026-05-13T13:34:50.259Z" }, +] + +[[package]] +name = "setuptools" +version = "81.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0d/1c/73e719955c59b8e424d015ab450f51c0af856ae46ea2da83eba51cc88de1/setuptools-81.0.0.tar.gz", hash = "sha256:487b53915f52501f0a79ccfd0c02c165ffe06631443a886740b91af4b7a5845a", size = 1198299, upload-time = "2026-02-06T21:10:39.601Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/e3/c164c88b2e5ce7b24d667b9bd83589cf4f3520d97cad01534cd3c4f55fdb/setuptools-81.0.0-py3-none-any.whl", hash = "sha256:fdd925d5c5d9f62e4b74b30d6dd7828ce236fd6ed998a08d81de62ce5a6310d6", size = 1062021, upload-time = "2026-02-06T21:10:37.175Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "smmap" +version = "5.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1f/ea/49c993d6dfdd7338c9b1000a0f36817ed7ec84577ae2e52f890d1a4ff909/smmap-5.0.3.tar.gz", hash = "sha256:4d9debb8b99007ae47165abc08670bd74cb74b5227dda7f643eccc4e9eb5642c", size = 22506, upload-time = "2026-03-09T03:43:26.1Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/d4/59e74daffcb57a07668852eeeb6035af9f32cbfd7a1d2511f17d2fe6a738/smmap-5.0.3-py3-none-any.whl", hash = "sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f", size = 24390, upload-time = "2026-03-09T03:43:24.361Z" }, +] + +[[package]] +name = "sympy" +version = "1.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mpmath" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, +] + +[[package]] +name = "torch" +version = "2.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-bindings", marker = "sys_platform == 'linux'" }, + { name = "cuda-toolkit", extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux'" }, + { name = "filelock" }, + { name = "fsspec" }, + { name = "jinja2" }, + { name = "networkx" }, + { name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cusparselt-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux'" }, + { name = "setuptools" }, + { name = "sympy" }, + { name = "triton", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/0d/98b410492609e34a155fa8b121b55c7dca229f39636851c3a9ec20edea21/torch-2.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7b6a60d48062809f58595509c524b88e6ddec3ebe25833d6462eeab81e5f2ce4", size = 80529712, upload-time = "2026-03-23T18:12:02.608Z" }, + { url = "https://files.pythonhosted.org/packages/84/03/acea680005f098f79fd70c1d9d5ccc0cb4296ec2af539a0450108232fc0c/torch-2.11.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:d91aac77f24082809d2c5a93f52a5f085032740a1ebc9252a7b052ef5a4fddc6", size = 419718178, upload-time = "2026-03-23T18:10:46.675Z" }, + { url = "https://files.pythonhosted.org/packages/8c/8b/d7be22fbec9ffee6cff31a39f8750d4b3a65d349a286cf4aec74c2375662/torch-2.11.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:7aa2f9bbc6d4595ba72138026b2074be1233186150e9292865e04b7a63b8c67a", size = 530604548, upload-time = "2026-03-23T18:10:03.569Z" }, + { url = "https://files.pythonhosted.org/packages/d1/bd/9912d30b68845256aabbb4a40aeefeef3c3b20db5211ccda653544ada4b6/torch-2.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:73e24aaf8f36ab90d95cd1761208b2eb70841c2a9ca1a3f9061b39fc5331b708", size = 114519675, upload-time = "2026-03-23T18:11:52.995Z" }, +] + +[[package]] +name = "torchvision" +version = "0.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "pillow" }, + { name = "torch" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/bd/d552a2521bade3295b2c6e7a4a0d1022261cab7ca7011f4e2a330dbb3caa/torchvision-0.26.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:55bd6ad4ae77be01ba67a410b05b51f53b0d0ee45f146eb6a0dfb9007e70ab3c", size = 1863499, upload-time = "2026-03-23T18:12:58.696Z" }, + { url = "https://files.pythonhosted.org/packages/33/bf/21b899792b08cae7a298551c68398a79e333697479ed311b3b067aab4bdc/torchvision-0.26.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:1c55dc8affbcc0eb2060fbabbe996ae9e5839b24bb6419777f17848945a411b1", size = 7767527, upload-time = "2026-03-23T18:12:44.348Z" }, + { url = "https://files.pythonhosted.org/packages/9a/45/57bbf9e216850d065e66dd31a50f57424b607f1d878ab8956e56a1f4e36b/torchvision-0.26.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:fd10b5f994c210f4f6d6761cf686f82d748554adf486cb0979770c3252868c8f", size = 7519925, upload-time = "2026-03-23T18:12:53.283Z" }, + { url = "https://files.pythonhosted.org/packages/10/58/ed8f7754299f3e91d6414b6dc09f62b3fa7c6e5d63dfe48d69ab81498a37/torchvision-0.26.0-cp311-cp311-win_amd64.whl", hash = "sha256:de6424b12887ad884f39a0ee446994ae3cd3b6a00a9cafe1bead85a031132af0", size = 3983834, upload-time = "2026-03-23T18:13:00.224Z" }, +] + +[[package]] +name = "tqdm" +version = "4.67.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/09/a9/6ba95a270c6f1fbcd8dac228323f2777d886cb206987444e4bce66338dd4/tqdm-4.67.3.tar.gz", hash = "sha256:7d825f03f89244ef73f1d4ce193cb1774a8179fd96f31d7e1dcde62092b960bb", size = 169598 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374 }, +] + +[[package]] +name = "triton" +version = "3.6.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0f/2c/96f92f3c60387e14cc45aed49487f3486f89ea27106c1b1376913c62abe4/triton-3.6.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49df5ef37379c0c2b5c0012286f80174fcf0e073e5ade1ca9a86c36814553651", size = 176081190, upload-time = "2026-01-20T16:16:00.523Z" }, + { url = "https://files.pythonhosted.org/packages/e0/12/b05ba554d2c623bffa59922b94b0775673de251f468a9609bc9e45de95e9/triton-3.6.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8e323d608e3a9bfcc2d9efcc90ceefb764a82b99dea12a86d643c72539ad5d3", size = 188214640, upload-time = "2026-01-20T16:00:35.869Z" }, +] + +[[package]] +name = "types-protobuf" +version = "7.34.1.20260518" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/29/59/e2b13b499d15e6720150c4b1a8d91e31fcacf716b432397475b3151ff7e4/types_protobuf-7.34.1.20260518.tar.gz", hash = "sha256:28cfaded25889cb83ebfb63cfb0a43628f0b6f3785767bec17287dc6468795f2", size = 68936 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/1f/ec5caf72c2e3b688ca3927e0979a04ddad19e1afc4bf1c199bd743e0f419/types_protobuf-7.34.1.20260518-py3-none-any.whl", hash = "sha256:a0a5337413347166439c0e07cbc26c6164d091401c6f01b1dfd8cdb966c4dd8f", size = 85992 }, +] + +[[package]] +name = "types-protobuf" +version = "7.34.1.20260518" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/29/59/e2b13b499d15e6720150c4b1a8d91e31fcacf716b432397475b3151ff7e4/types_protobuf-7.34.1.20260518.tar.gz", hash = "sha256:28cfaded25889cb83ebfb63cfb0a43628f0b6f3785767bec17287dc6468795f2", size = 68936 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/1f/ec5caf72c2e3b688ca3927e0979a04ddad19e1afc4bf1c199bd743e0f419/types_protobuf-7.34.1.20260518-py3-none-any.whl", hash = "sha256:a0a5337413347166439c0e07cbc26c6164d091401c6f01b1dfd8cdb966c4dd8f", size = 85992 }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] + +[[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, +] + +[[package]] +name = "wandb" +version = "0.27.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "gitpython" }, + { name = "packaging" }, + { name = "platformdirs" }, + { name = "protobuf" }, + { name = "pydantic" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "sentry-sdk" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8e/31/fe53d06b75ef0a7f2f0ee5931a89f7aedc27d233840b1839616860fed256/wandb-0.27.0.tar.gz", hash = "sha256:579e75300173059f9334e1f513a79ef15f6d9ea5c74e20d695633648cdd02031", size = 41090732, upload-time = "2026-05-14T03:44:08.894Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/5e/2c199e70e636ecfd217cde0bc7469f4511e1d03d0685eb92bfdfce391430/wandb-0.27.0-py3-none-macosx_12_0_arm64.whl", hash = "sha256:c156be4851485f3c4160cb6eb2e8991b4cdeffbccefc5636d33cf5e254847365", size = 24886476, upload-time = "2026-05-14T03:43:27.569Z" }, + { url = "https://files.pythonhosted.org/packages/0b/cd/a617c871cd304a9804e56a7ec2ec2c65685bf0091a2b9f91910175a149e2/wandb-0.27.0-py3-none-macosx_12_0_x86_64.whl", hash = "sha256:20179f38afb0158859a4141d29ac650d3fdbd0cf801a74ce25565c934f03776c", size = 26045779, upload-time = "2026-05-14T03:43:31.999Z" }, + { url = "https://files.pythonhosted.org/packages/10/0a/d3f159a201530b84b72ca5f98c68d1f351c2d9a1864558ed76c811407fae/wandb-0.27.0-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:626497d7975fa898d0a4a239da7a510483495ca3514510dbe75004a25963af4d", size = 25480764, upload-time = "2026-05-14T03:43:35.922Z" }, + { url = "https://files.pythonhosted.org/packages/5f/6a/8721fcdf71d42639191040a77a585d2982402b1754700cb2ecfc2ca1470a/wandb-0.27.0-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:f772da7005cc26a2a32b729a16982a583dc68b3d493df6a09d0aa5c5ca5a2060", size = 27256204, upload-time = "2026-05-14T03:43:39.765Z" }, + { url = "https://files.pythonhosted.org/packages/00/5e/279d167ba79fb7a8a43401c9f25efd0f6663ee9bd1eaf5a8578530198888/wandb-0.27.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:63acfc5b994e4a90e4a2fbdee6d45e664da3dd865bb1419942c8995c06c41cf1", size = 25647469, upload-time = "2026-05-14T03:43:44.817Z" }, + { url = "https://files.pythonhosted.org/packages/94/51/a69ac59300e3c813939d0764348959ed2a21e14c668cb1cebcb04010da6a/wandb-0.27.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:17aae6e4a88cd05c00ea8f546220918e3ebb6f8c1c36b70ef04a5ac75f0d7160", size = 27599005, upload-time = "2026-05-14T03:43:50.926Z" }, + { url = "https://files.pythonhosted.org/packages/5f/40/bf510c8758727df020f83b717ebc1fcc1739ed7f6ae1796ebef60bf6f592/wandb-0.27.0-py3-none-win32.whl", hash = "sha256:0bd5659417e386bf6538b5e2ffe6885774c6197f0e4853bfed517d5b0db457f1", size = 25036164, upload-time = "2026-05-14T03:43:54.839Z" }, + { url = "https://files.pythonhosted.org/packages/54/ff/69f88e7d90c22b79bcb911143c13e59742ee192080b21015ff83a5a1f60a/wandb-0.27.0-py3-none-win_amd64.whl", hash = "sha256:89d584b73166eecee96fb446f18d0e45b1aa45aba6a3696296f3f06d7454516b", size = 25036170, upload-time = "2026-05-14T03:43:59.227Z" }, + { url = "https://files.pythonhosted.org/packages/f6/38/f7efd7a87297a55c7e9a331a1dbb5b19e54aeacc11fe6f43f8636a73987c/wandb-0.27.0-py3-none-win_arm64.whl", hash = "sha256:a6c129c311edf210a2b4f2f4acc557eff522628125f5f28ed27df19c16c07079", size = 22972710, upload-time = "2026-05-14T03:44:03.275Z" }, +] + +[[package]] +name = "wrapt" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2d/9f/06263fcd8ad6c405f05a3905fd7a84dd3176eb5ad46e44bccc0cd16348bb/wrapt-2.2.1.tar.gz", hash = "sha256:6744f504375775d7609c82c8d3d94af1c9a6f05586984536905908ba905277b9", size = 127620 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/ac/4370bde262c0e633e6c4f0e56d55095710024cf9a5cecc20c59a10de483c/wrapt-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:dd57607acc85678925940bd5df0385ff8332083a32fa8d7a43f8767f4997263c", size = 80321 }, + { url = "https://files.pythonhosted.org/packages/eb/79/b8ff3a61e71babf58a8cf4c0d63358e8bad383e15bf7f35e62d2f6b6e4a4/wrapt-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1ae574d65c9fa8e86f64f6a7c2668f9fcd507b183e0e577619f504b883cb0a6c", size = 81216 }, + { url = "https://files.pythonhosted.org/packages/6e/fd/c0cac1f77c9c4f6fe58a920ca632ce379bb8be928720e11e8d73de28a5e9/wrapt-2.2.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9a04c28c10ba7fd12842b109d2edb0678872a2fe65277ca4ff06a0d61edee245", size = 159208 }, + { url = "https://files.pythonhosted.org/packages/d9/4f/744132a7b2fbefa6b81118ec5942eca5fc2e9a129f9055a0c5e46885a549/wrapt-2.2.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3e2f02472a1cbbf3884b365714a810b5947134a95ad6952b554cb8cce9d492b0", size = 160322 }, + { url = "https://files.pythonhosted.org/packages/d6/95/b7cd9a22a06cf93e6482904ee6afc956248983553593fd1009296d1b3b31/wrapt-2.2.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ac2745950b2bff80219c15ebf2fa9d8427eba7e249739f97e55c9d169e47e9e1", size = 153243 }, + { url = "https://files.pythonhosted.org/packages/4c/4a/eb79423192015f46f0db2872e7e04a3dde8d359b83411e8959e7c9287eaa/wrapt-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:67a97e5b6c457f0cd3cfc19ebb2d84463e60c3ece754cc831e4281a3ca29bb18", size = 159231 }, + { url = "https://files.pythonhosted.org/packages/ec/dc/435015b58ce33c6fc4104158fa91ddb0e809ab03a5751fb7465d1d461456/wrapt-2.2.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:c803a3d331796255af51ba2c79ed0ac8275865b516c09e61f248d1e7aff31ce9", size = 152351 }, + { url = "https://files.pythonhosted.org/packages/77/ac/5d203f98df8fd136b95c5227139aea02d34505e18baf812d0c005df61963/wrapt-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9b984d1eb252145d6302c1dbd5e87fc6d404d45531447c84eadec04bf1fcb027", size = 158347 }, + { url = "https://files.pythonhosted.org/packages/52/2f/a92427dbdc74e54c1674abbed27e61b2cb5e7a94441b8c1270c70671d928/wrapt-2.2.1-cp311-cp311-win32.whl", hash = "sha256:8a983a603a18c8708f024f7f6991b2e66159219abbf894634c5056243c55f3cd", size = 77562 }, + { url = "https://files.pythonhosted.org/packages/c8/56/987b9c13b3e1c1a3c6de71284076f996b79caec90e75a87c044a40c23db9/wrapt-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:9c210a6994b21aa9b29e81c8d11560e8fdab54c117e9cff37870d0a27bde1343", size = 80616 }, + { url = "https://files.pythonhosted.org/packages/7e/25/d01f560888d99d94a959c85533de349ce68d71ace3f2591d6ea8f632cfed/wrapt-2.2.1-cp311-cp311-win_arm64.whl", hash = "sha256:401229e9d63ca09f9b8891ecf83798d26c11bbb445d11ed9f1836b6d4585b38a", size = 79025 }, + { url = "https://files.pythonhosted.org/packages/53/46/29ac9daf11a86c22a8c38cd9236c62928ccae83f7ceb06bd3b0467cf9d05/wrapt-2.2.1-py3-none-any.whl", hash = "sha256:3aafea2975caef8ca49400640dde02cc7426e798f24870ed01f490bc3cffd32f", size = 61000 }, +] + +[[package]] +name = "wrapt" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2d/9f/06263fcd8ad6c405f05a3905fd7a84dd3176eb5ad46e44bccc0cd16348bb/wrapt-2.2.1.tar.gz", hash = "sha256:6744f504375775d7609c82c8d3d94af1c9a6f05586984536905908ba905277b9", size = 127620 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/ac/4370bde262c0e633e6c4f0e56d55095710024cf9a5cecc20c59a10de483c/wrapt-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:dd57607acc85678925940bd5df0385ff8332083a32fa8d7a43f8767f4997263c", size = 80321 }, + { url = "https://files.pythonhosted.org/packages/eb/79/b8ff3a61e71babf58a8cf4c0d63358e8bad383e15bf7f35e62d2f6b6e4a4/wrapt-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1ae574d65c9fa8e86f64f6a7c2668f9fcd507b183e0e577619f504b883cb0a6c", size = 81216 }, + { url = "https://files.pythonhosted.org/packages/6e/fd/c0cac1f77c9c4f6fe58a920ca632ce379bb8be928720e11e8d73de28a5e9/wrapt-2.2.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9a04c28c10ba7fd12842b109d2edb0678872a2fe65277ca4ff06a0d61edee245", size = 159208 }, + { url = "https://files.pythonhosted.org/packages/d9/4f/744132a7b2fbefa6b81118ec5942eca5fc2e9a129f9055a0c5e46885a549/wrapt-2.2.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3e2f02472a1cbbf3884b365714a810b5947134a95ad6952b554cb8cce9d492b0", size = 160322 }, + { url = "https://files.pythonhosted.org/packages/d6/95/b7cd9a22a06cf93e6482904ee6afc956248983553593fd1009296d1b3b31/wrapt-2.2.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ac2745950b2bff80219c15ebf2fa9d8427eba7e249739f97e55c9d169e47e9e1", size = 153243 }, + { url = "https://files.pythonhosted.org/packages/4c/4a/eb79423192015f46f0db2872e7e04a3dde8d359b83411e8959e7c9287eaa/wrapt-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:67a97e5b6c457f0cd3cfc19ebb2d84463e60c3ece754cc831e4281a3ca29bb18", size = 159231 }, + { url = "https://files.pythonhosted.org/packages/ec/dc/435015b58ce33c6fc4104158fa91ddb0e809ab03a5751fb7465d1d461456/wrapt-2.2.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:c803a3d331796255af51ba2c79ed0ac8275865b516c09e61f248d1e7aff31ce9", size = 152351 }, + { url = "https://files.pythonhosted.org/packages/77/ac/5d203f98df8fd136b95c5227139aea02d34505e18baf812d0c005df61963/wrapt-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9b984d1eb252145d6302c1dbd5e87fc6d404d45531447c84eadec04bf1fcb027", size = 158347 }, + { url = "https://files.pythonhosted.org/packages/52/2f/a92427dbdc74e54c1674abbed27e61b2cb5e7a94441b8c1270c70671d928/wrapt-2.2.1-cp311-cp311-win32.whl", hash = "sha256:8a983a603a18c8708f024f7f6991b2e66159219abbf894634c5056243c55f3cd", size = 77562 }, + { url = "https://files.pythonhosted.org/packages/c8/56/987b9c13b3e1c1a3c6de71284076f996b79caec90e75a87c044a40c23db9/wrapt-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:9c210a6994b21aa9b29e81c8d11560e8fdab54c117e9cff37870d0a27bde1343", size = 80616 }, + { url = "https://files.pythonhosted.org/packages/7e/25/d01f560888d99d94a959c85533de349ce68d71ace3f2591d6ea8f632cfed/wrapt-2.2.1-cp311-cp311-win_arm64.whl", hash = "sha256:401229e9d63ca09f9b8891ecf83798d26c11bbb445d11ed9f1836b6d4585b38a", size = 79025 }, + { url = "https://files.pythonhosted.org/packages/53/46/29ac9daf11a86c22a8c38cd9236c62928ccae83f7ceb06bd3b0467cf9d05/wrapt-2.2.1-py3-none-any.whl", hash = "sha256:3aafea2975caef8ca49400640dde02cc7426e798f24870ed01f490bc3cffd32f", size = 61000 }, +]