lansinuote / simple_reinforcement_learning Goto Github PK

def reset(self):
    state, _ = self.env.reset()
    self.step_n = 0
    return state

def step(self, action):
    state, reward, terminated, truncated, info = self.env.step(action)
    done = terminated or truncated
    self.step_n += 1
    if self.step_n >= 200:
        done = True
    return state, reward, done, info

prob = model_actor(state)

action = random.choices(range(11), weights=prob[0].tolist(),k=1)[0]
action_continuous = action
action_continuous /= 10
action_continuous *= 4
action_continuous -= 2

return action, action_continuous

# 初始化游戏
state = env.reset()
over = False
while not over:
    action, action_continuous = get_action(state)
    #print(f'执行动作:{action_continuous}')
    next_state, reward, over, _ = env.step([action_continuous])
    states.append(state)
    rewards.append(reward)
    actions.append(action)
    next_states.append(next_state)
    overs.append(over)

    state = next_state

states = torch.FloatTensor(states).reshape(-1, 3)

rewards = torch.FloatTensor(rewards).reshape(-1, 1)

actions = torch.LongTensor(actions).reshape(-1, 1)

next_states = torch.FloatTensor(next_states).reshape(-1, 3)

overs = torch.LongTensor(overs).reshape(-1, 1)

return states, rewards, actions, next_states, overs

#记录反馈值的和,这个值越大越好
reward_sum = 0

#玩到游戏结束为止
over = False

while not over:
    
    #根据当前状态得到一个动作
    action,action_continuous = get_action(state)

    #执行动作,得到反馈
    state, reward, over, _ = env.step([action_continuous])
    reward_sum += reward

    #打印动画
    if play: 
        display.clear_output(wait=True)
        show()

return reward_sum

loss_fn = torch.nn.MSELoss()

for i in range(2000):
    states, rewards, actions, next_states, overs = get_data()

    values = model_critic(states)

    targets = model_critic(next_states)

    targets *= 0.98

    targets *= 1 - overs

    targets += rewards

    delta = (values - targets).detach()

    probs = model_actor(states)

    probs = probs.gather(dim=1, index=actions)

    loss = (-probs.log() * delta).mean()

    loss_critic =loss_fn(values,targets.detach())

    optimizer_actor.zero_grad()
    loss.backward()
    optimizer_actor.step()

    optimizer_critic.zero_grad()
    loss_critic.backward()
    optimizer_critic.step()

    if i % 100 == 0:
        test_result = sum([test(play=False) for _ in range(10)]) / 10
        print(f"epoch:{i},score:{test_result}")