Я пытаюсь построить новое наблюдение и применить его в DQN.
Я использую алгоритм Pytorch DQN с моим кодом наблюдения.
это еще не закончено, поэтому данные не ясно
Я изменил некоторую строку во всем коде следующим образом.
import sys, math
import random as rd
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
######################################
from collections import namedtuple
from itertools import count
from PIL import Image
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torch.autograd import Variable
import torchvision.transforms as T
Transition = namedtuple('Transition',
('state', 'action', 'next_state', 'reward'))
class ReplayMemory(object):
def __init__(self, capacity):
self.capacity = capacity
self.memory = []
self.position = 0
def push(self, *args):
"""Saves a transition."""
if len(self.memory) < self.capacity:
self.memory.append(None)
self.memory[self.position] = Transition(*args)
self.position = (self.position + 1) % self.capacity
def sample(self, batch_size):
return rd.sample(self.memory, batch_size)
def __len__(self):
return len(self.memory)
class DQN(nn.Module):
def __init__(self):
super(DQN, self).__init__()
self.l1 = nn.Linear(5, 16)
self.l2 = nn.Linear(16, 12)
self.l3 = nn.Linear(12, 20)
self.head = nn.Linear(20, 1)
def forward(self, x):
x = F.relu(self.l1(x))
x = F.relu(self.l2(x))
x = F.relu(self.l3(x))
return self.head(x.view(x.size(0), -1))
BATCH_SIZE = 5
GAMMA = 0.999
EPS_START = 0.9
EPS_END = 0.05
EPS_DECAY = 200
TARGET_UPDATE = 5
policy_net = DQN()
target_net = DQN()
target_net.load_state_dict(policy_net.state_dict())
target_net.eval()
optimizer = optim.RMSprop(policy_net.parameters())
memory = ReplayMemory(10000)
def optimize_model():
if len(memory) < BATCH_SIZE:
return
transitions = memory.sample(BATCH_SIZE)
# Transpose the batch (see http://stackoverflow.com/a/19343/3343043 for
# detailed explanation).
batch = Transition(*zip(*transitions))
print("batch = ", batch, "\n")
# Compute a mask of non-final states and concatenate the batch elements
state_batch = Variable(torch.cat(batch.state))
print("state_batch = ", state_batch)
action_batch = Variable(torch.cat(batch.action))
print("action_batch = ", action_batch)
reward_batch = Variable(torch.cat(batch.reward), requires_grad = False)
print("reward_batch = ", reward_batch)
next_state_batch = Variable(torch.cat(batch.next_state), requires_grad = False)
# Compute Q(s_t, a) - the model computes Q(s_t), then we select the
# columns of actions taken
state_action_values = policy_net(state_batch)
print("state_action_values = ", state_action_values)
# Compute V(s_{t+1}) for all next states.
next_state_values = target_net(next_state_batch)
print("next_state_values = ", next_state_values)
# Compute the expected Q values
expected_state_action_values = (next_state_values * GAMMA) + reward_batch
print("expected next state values = ", expected_state_action_values)
# Compute Huber loss
loss = F.smooth_l1_loss(state_action_values, expected_state_action_values, reduce = False)
# Optimize the model
optimizer.zero_grad()
loss.backward()
for param in policy_net.parameters():
param.grad.data.clamp_(-1, 1)
optimizer.step()
num_episodes = 5
for i_episode in range(num_episodes):
# Initialize the environment and state
drive = AutoDrive(20, 20, 0, 16, 0) # x/y/yaw/velocity/heading
drive._make_observation(0, -1, -1, -1, -1, -1) random other vehicle location, parameters
stand = 3
# exploit 1
e = 1. / ((i_episode // 100) + 1) # conditioon for choosing action
optimizer.zero_grad()
for t in range(stand):
# Select and perform an action
# exploit 2
if np.random.rand(1) > e:
action = rd.randint(1, 4)
else:
action = np.argmax(drive._select_action(0.5, 0.5)) + 1 #index + 1
print("state = ", drive.state, ", action = ", action, ", yaw = ", drive.yaw, ", mag = ", drive.mag)
state = drive.state
drive._step(action)
drive._calc_reward(0.5, 0.5)
print(drive.reward)
if (drive.reward == -10):
break
# Store the transition in memory
state1 = torch.FloatTensor(state).view(1, 5)
state2 = torch.FloatTensor(drive.state).view(1, 5)
action = torch.FloatTensor([float(action)]).view(1, 1)
reward = torch.FloatTensor([drive.reward]).view(1, 1)
memory.push(state1, action\
, state2, reward)
# Perform one step of the optimization (on the target network)
optimize_model()
if done:
episode_durations.append(t + 1)
plot_durations()
break
# Update the target network
if i_episode % TARGET_UPDATE == 0:
target_net.load_state_dict(policy_net.state_dict())
ошибка возникает в функции потерь
File "<ipython-input-190-29dcdbbf0383>", line 1, in <module>
runfile('C:/Users/desktop/.spyder-py3/temp.py', wdir='C:/Users/desktop/.spyder-py3')
File "C:\ProgramData\Anaconda3\lib\site-packages\spyder\utils\site\sitecustomize.py", line 705, in runfile
execfile(filename, namespace)
File "C:\ProgramData\Anaconda3\lib\site-packages\spyder\utils\site\sitecustomize.py", line 102, in execfile
exec(compile(f.read(), filename, 'exec'), namespace)
File "C:/Users/desktop/.spyder-py3/temp.py", line 441, in <module>
optimize_model()
File "C:/Users/desktop/.spyder-py3/temp.py", line 362, in optimize_model
loss = F.smooth_l1_loss(state_action_values, expected_state_action_values, reduce = False)
RuntimeError: the derivative for 'target' is not implemented
входы в функции следующие.
expected next state values = Variable containing:
8.9615
12.0198
12.0488
12.2920
13.9062
[torch.FloatTensor of size 5x1]
state_action_values = Variable containing:
0.3765
0.5196
0.4587
0.3765
0.5636
[torch.FloatTensor of size 5x1]
что мне делать? я действительно начинающий, так что полезные советы приветствуются