i am Getting Started with RLlib and sumo i am tring a rl algorithm to manage traffic lights here is the code
description of the code :
This code defines a reinforcement learning environment for traffic signal control using SUMO (Simulation of Urban MObility) traffic simulation software and OpenAI Gym framework. The environment is defined as a class called "SumoEnv" that extends the "gym.Env" class. It defines the methods "reset", "step", "get_observation", "get_reward", and "is_done", which are the standard methods required in a Gym environment.
The "reset" method initializes the environment and returns the initial observation. The "step" method takes an action as an input, updates the simulation accordingly, and returns the new observation, reward, and whether the episode is done or not. The "get_observation" method returns the observation of the current state of the environment, which is the number of vehicles in each lane. The "get_reward" method calculates the reward of the current state based on the waiting time of vehicles in each lane. The "is_done" method returns whether the simulation should be terminated or not, which is the case when there are no more vehicles in the simulation.
The code also uses Ray and Ray Tune to train a reinforcement learning agent using the Proximal Policy Optimization (PPO) algorithm to optimize the traffic signal control policy in this environment. The configuration of the PPO algorithm is defined in the "config" dictionary.
import gymnasium as gym
import numpy as np
from gym import spaces
import traci
from collections.abc import Mapping
class SumoEnv(gym.Env):
def __init__(self, env_config):
self.sumo_binary = env_config\["sumo_binary"\]
self.sumo_cmd = \[self.sumo_binary, "-c", env_config\["sumocfg"\]\]
traci.start(self.sumo_cmd)
self.action_map = {"G": 'GGgrrrGGgrrr', "R": 'rrrGGgrrrGGg'}
self.observation_space = spaces.Box(low=0, high=np.inf, shape=(env_config["num_lanes"],))
self.action_space = spaces.Discrete(2)
def reset(self):
traci.load(self.sumo_cmd[1:])
return self.get_observation()
def step(self, action):
action = int(action)
action = ["G", "R"][action]
traci.simulationStep()
traci.trafficlight.setPhase("tl", self.action_map[action])
obs = self.get_observation()
reward = self.get_reward()
done = self.is_done()
return obs, reward, done, {}
def get_observation(self):
lanes = traci.lane.getIDList()
obs = [traci.lane.getLastStepVehicleNumber(lane) for lane in lanes]
return np.array(obs)
def get_reward(self):
lanes = traci.lane.getIDList()
reward = -sum([traci.lane.getWaitingTime(lane) for lane in lanes])
return reward
def is_done(self):
done = (traci.simulation.getMinExpectedNumber() <= 0)
return done
import ray
from ray import tune
from ray.rllib.agents.ppo import PPOTrainer
ray.init()
env_config = {
"sumo_binary": 'C:\\Program Files (x86)\\Eclipse\\Sumo\\bin\\sumo-gui.exe',
"sumocfg": "configuration.sumocfg",
"num_lanes": 8
}
config = {
"env": SumoEnv,
"env_config": env_config,
"num_workers": 1,
"num_gpus": 0,
"lr": 1e-4,
}
tune.run(PPOTrainer, config=config)
but the gui lanches and nothing happens no learning is being done
i tried relaunching multiple time or klicking the start button from the gui but it just shuts down and in ray_results this the error i get
Failure # 1 (occurred at 2023-04-05_01-14-34)
Traceback (most recent call last):
File "C:\Users\med\anaconda3\lib\site-packages\ray\tune\execution\ray_trial_executor.py", line 1276, in get_next_executor_event
future_result = ray.get(ready_future)
File "C:\Users\med\anaconda3\lib\site-packages\ray\_private\client_mode_hook.py", line 105, in wrapper
return func(*args, **kwargs)
File "C:\Users\med\anaconda3\lib\site-packages\ray\_private\worker.py", line 2382, in get
raise value
File "python\ray\_raylet.pyx", line 1166, in ray._raylet.task_execution_handler
File "python\ray\_raylet.pyx", line 1072, in ray._raylet.execute_task_with_cancellation_handler
File "python\ray\_raylet.pyx", line 805, in ray._raylet.execute_task
File "python\ray\_raylet.pyx", line 972, in ray._raylet.execute_task
File "python\ray\_raylet.pyx", line 621, in ray._raylet.store_task_errors
ray.exceptions.RayActorError: The actor died because of an error raised in its creation task, [36mray::PPO.__init__()[39m (pid=25932, ip=127.0.0.1, repr=PPO)
File "C:\Users\med\anaconda3\lib\site-packages\ray\rllib\evaluation\worker_set.py", line 240, in _setup
self.add_workers(
File "C:\Users\med\anaconda3\lib\site-packages\ray\rllib\evaluation\worker_set.py", line 614, in add_workers
raise result.get()
File "C:\Users\med\anaconda3\lib\site-packages\ray\rllib\utils\actor_manager.py", line 477, in __fetch_result
result = ray.get(r)
File "C:\Users\med\anaconda3\lib\site-packages\ray\_private\client_mode_hook.py", line 105, in wrapper
return func(*args, **kwargs)
File "C:\Users\med\anaconda3\lib\site-packages\ray\_private\worker.py", line 2382, in get
raise value
File "python\ray\_raylet.pyx", line 1166, in ray._raylet.task_execution_handler
File "python\ray\_raylet.pyx", line 1072, in ray._raylet.execute_task_with_cancellation_handler
File "python\ray\_raylet.pyx", line 805, in ray._raylet.execute_task
File "python\ray\_raylet.pyx", line 972, in ray._raylet.execute_task
File "python\ray\_raylet.pyx", line 621, in ray._raylet.store_task_errors
ray.exceptions.RayActorError: The actor died because of an error raised in its creation task, [36mray::RolloutWorker.__init__()[39m (pid=15664, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x000001C3CAB0A070>)
TypeError: reset() got an unexpected keyword argument 'seed'
During handling of the above exception, another exception occurred:
[36mray::RolloutWorker.__init__()[39m (pid=15664, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x000001C3CAB0A070>)
File "python\ray\_raylet.pyx", line 850, in ray._raylet.execute_task
File "python\ray\_raylet.pyx", line 902, in ray._raylet.execute_task
File "python\ray\_raylet.pyx", line 857, in ray._raylet.execute_task
File "python\ray\_raylet.pyx", line 861, in ray._raylet.execute_task
File "python\ray\_raylet.pyx", line 803, in ray._raylet.execute_task.function_executor
File "C:\Users\med\anaconda3\lib\site-packages\ray\_private\function_manager.py", line 674, in actor_method_executor
return method(__ray_actor, *args, **kwargs)
File "C:\Users\med\anaconda3\lib\site-packages\ray\util\tracing\tracing_helper.py", line 466, in _resume_span
return method(self, *_args, **_kwargs)
File "C:\Users\med\anaconda3\lib\site-packages\ray\rllib\evaluation\rollout_worker.py", line 607, in __init__
self.env = env_creator(copy.deepcopy(self.env_context))
File "C:\Users\med\anaconda3\lib\site-packages\ray\rllib\env\utils.py", line 175, in _gym_env_creator
obs_and_infos = env.reset(seed=None, options={})
File "C:\Users\med\anaconda3\lib\site-packages\gymnasium\wrappers\compatibility.py", line 100, in reset
return self.env.reset(), {}
File "C:\Users\med\AppData\Local\Temp\ipykernel_21904\393336602.py", line 22, in reset
File "C:\Users\med\AppData\Local\Temp\ipykernel_21904\393336602.py", line 41, in get_observation
File "C:\Program Files (x86)\Eclipse\Sumo\tools\traci\domain.py", line 171, in getIDList
return self._getUniversal(tc.TRACI_ID_LIST, "")
File "C:\Program Files (x86)\Eclipse\Sumo\tools\traci\domain.py", line 147, in _getUniversal
return _parse(self._retValFunc, varID, self._getCmd(varID, objectID, format, *values))
File "C:\Program Files (x86)\Eclipse\Sumo\tools\traci\domain.py", line 152, in _getCmd
r = self._connection._sendCmd(self._cmdGetID, varID, objID, format, *values)
File "C:\Program Files (x86)\Eclipse\Sumo\tools\traci\connection.py", line 225, in _sendCmd
return self._sendExact()
File "C:\Program Files (x86)\Eclipse\Sumo\tools\traci\connection.py", line 144, in _sendExact
raise FatalTraCIError("Received answer %s for command %s." % (prefix[1], command))
traci.exceptions.FatalTraCIError: Received answer 2 for command 163.
During handling of the above exception, another exception occurred:
[36mray::PPO.__init__()[39m (pid=25932, ip=127.0.0.1, repr=PPO)
File "python\ray\_raylet.pyx", line 850, in ray._raylet.execute_task
File "python\ray\_raylet.pyx", line 902, in ray._raylet.execute_task
File "python\ray\_raylet.pyx", line 857, in ray._raylet.execute_task
File "python\ray\_raylet.pyx", line 861, in ray._raylet.execute_task
File "python\ray\_raylet.pyx", line 803, in ray._raylet.execute_task.function_executor
File "C:\Users\med\anaconda3\lib\site-packages\ray\_private\function_manager.py", line 674, in actor_method_executor
return method(__ray_actor, *args, **kwargs)
File "C:\Users\med\anaconda3\lib\site-packages\ray\util\tracing\tracing_helper.py", line 466, in _resume_span
return method(self, *_args, **_kwargs)
File "C:\Users\med\anaconda3\lib\site-packages\ray\rllib\algorithms\algorithm.py", line 445, in __init__
super().__init__(
File "C:\Users\med\anaconda3\lib\site-packages\ray\tune\trainable\trainable.py", line 169, in __init__
self.setup(copy.deepcopy(self.config))
File "C:\Users\med\anaconda3\lib\site-packages\ray\util\tracing\tracing_helper.py", line 466, in _resume_span
return method(self, *_args, **_kwargs)
File "C:\Users\med\anaconda3\lib\site-packages\ray\rllib\algorithms\algorithm.py", line 571, in setup
self.workers = WorkerSet(
File "C:\Users\med\anaconda3\lib\site-packages\ray\rllib\evaluation\worker_set.py", line 192, in __init__
raise e.args[0].args[2]
traci.exceptions.FatalTraCIError: Received answer 2 for command 163.
