Nrthugu

I wrote a Q-Learning implementation to solve OpenAI FrozenLake-v0 problem using simple NN.

My neural network looks like that:

Input layer: 16

Output layer: 4

Implementation in clear tensorflow did very well. About 70% finished episodes after training of 10k episodes.

After that I wanted to write the same algorithm using Keras, but this time algorithm did very poorly, after 10k episodes about 5% finished episodes.

I'm guessing I made mistake in my Keras implementation but I cannot figure it out.

Tensorflow implementation:

import gym

import numpy as np

import tensorflow as tf



env = gym.make('FrozenLake-v0')



discount_rate = 0.99

random_action_chance = 0.1

num_episodes = 10000

max_episode_step = 100

log_interval = 100



tf.reset_default_graph()

inputs = tf.placeholder(shape=[1, 16], dtype=tf.float32)

W = tf.Variable(tf.random_uniform([16, 4], 0, 0.01))

Q = tf.matmul(inputs, W)

predict = tf.argmax(Q, 1)



Qnext = tf.placeholder(shape=[1, 4], dtype=tf.float32)

loss = tf.reduce_sum(tf.square(Qnext - Q))

optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.1)

updateModel = optimizer.minimize(loss)



init = tf.initialize_all_variables()

rewards_from_episodes = 

with tf.Session() as sess:

    sess.run(init)

    for episode in range(num_episodes):

        observation = env.reset()

        episode_reward = 0

        if episode % log_interval == 0 and episode > 0:

            print('Episode: {}, Reward: {}'.format(episode, sum(rewards_from_episodes[episode - log_interval: episode]) / log_interval))



        W1 = 

        for step in range(max_episode_step):

            # Select action

            action, targetQ = sess.run([predict, Q], feed_dict={inputs: np.identity(16)[observation:observation + 1]})

            if np.random.rand(1) < random_action_chance:

                action[0] = env.action_space.sample()



            new_observation, reward, done, _ = env.step(action[0])

            Qnew = sess.run(Q, feed_dict={inputs: np.identity(16)[new_observation:new_observation + 1]})

            maxQvalue = np.max(Qnew)

            targetQ[0, action[0]] = reward + discount_rate * maxQvalue

            # Train network using target and predicted Q values

            _, W1 = sess.run([updateModel, W], feed_dict={inputs: np.identity(16)[observation:observation + 1],

                                                          Qnext: targetQ})

            episode_reward += reward

            observation = new_observation

            if done:

                random_action_chance = 1. / ((episode / 50) + 10)

                break

        rewards_from_episodes.append(episode_reward)



print("Mean of all episodes: {}%".format(sum(rewards_from_episodes) / num_episodes))

Keras implementation:

import gym

import numpy as np

import tensorflow as tf

import random

from tensorflow.python.keras.layers import *



env = gym.make('FrozenLake-v0')



learning_rate = 0.1

discount_rate = 0.99

random_action_chance = 0.1

num_episodes = 10000

max_episode_step = 100

log_interval = 100





model = tf.keras.Sequential()

model.add(Dense(4, kernel_initializer='uniform'))

model.compile(optimizer=tf.train.GradientDescentOptimizer(learning_rate=learning_rate),

              loss='mean_squared_error')



rewards_from_episodes = 



for episode in range(num_episodes):

    observation = env.reset()

    episode_reward = 0

    if episode % log_interval == 0 and episode > 0:

        print('Episode: {}, Reward: {}'.format(episode, sum(

            rewards_from_episodes[episode - log_interval: episode]) / log_interval))



    for step in range(max_episode_step):

        # Select action

        targetQ = model.predict(np.identity(16)[observation:observation + 1], batch_size=1)

        action = np.argmax(targetQ)



        if random.random() < random_action_chance:

            action = env.action_space.sample()



        new_observation, reward, done, _ = env.step(action)

        Qnew = model.predict(np.identity(16)[new_observation:new_observation + 1], batch_size=1)

        maxQvalue = np.max(Qnew)

        targetQ[0, action] = reward + discount_rate * maxQvalue



        # Train network using target and predicted Q values

        model.fit(np.identity(16)[observation:observation + 1], targetQ, epochs=1, batch_size=1, verbose=0)



        episode_reward += reward

        observation = new_observation

        if done:

            random_action_chance = 1. / ((episode / 50) + 10)

            break

    rewards_from_episodes.append(episode_reward)



print("Mean of all episodes: {}%".format(sum(rewards_from_episodes) / num_episodes))