Tukukkk

I'm trying to make a Q table based reinforcement learning algorithm play Connect Four against a Neural Network Q table. It seems to work, but when I try to use the Q table against a opponent that randomly picks moves, it loses almost every time.

I got the connect four code from this github page and modified it into a class with a couple of extra functions such that give a list of valid moves and such.

Here is my connect four class

And here is my Q table code

def trainQ(nRepetitions, learningRate, epsilonDecayFactor):

    # Initialize Q

    Qred = {}

    Qyellow = {}

    players = ['R', 'Y']

    steps = 

    epsilon = 1

    numTimesRWon = 0

    numTimesYWon = 0

    for step in range(nRepetitions):

        game = Game()

        sOld = None

        aOld = None

        stepCount = 0

        epsilon = epsilon * epsilonDecayFactor

        while not game.isDone():

             player = players[stepCount % 2 == 0]

             # Select next action.

             if player == 'R':

                 moves = game.validMoves()

                 # List of Q values for valid moves

                 moveQ = [Qred.get(game.boardTup(game.board, move), 0) for move in moves]

                 ranNum = np.random.random()

                 if epsilon > ranNum:

                     a = moves[np.random.choice(len(moveQ))]

                else:

                     a = moves[np.argmax(np.array(moveQ))]

                # If not first step, update Qold with TD error (1 + Qnew - Qold)

                if sOld != None:

                     Qred[game.boardTup(sOld, aOld)] = Qred.get(game.boardTup(sOld, aOld), 0) + learningRate * 

                                                (1 + Qred.get(game.boardTup(game.board, a), 0) - Qred.get(game.boardTup(sOld, aOld), 0))

                # Shift current board and action to old ones.

                sOld, aOld = game.board, a

                # Apply action to get new board.

                game.insert(a, player)

                stepCount += 1

            if player == 'Y':

                moves = game.validMoves()

                # List of Q values for valid moves

                moveQ = [Qyellow.get(game.boardTup(game.board, move), 0) for move in moves]

                ranNum = np.random.random()

                if True:#epsilon > ranNum:

                    a = moves[np.random.choice(len(moveQ))]

                else:

                    a = moves[np.argmax(np.array(moveQ))]

                # If not first step, update Qold with TD error (1 + Qnew - Qold)

                if sOld != None:

                    Qyellow[game.boardTup(sOld, aOld)] = Qyellow.get(game.boardTup(sOld, aOld), 0) + learningRate * 

                                                (1 + Qyellow.get(game.boardTup(game.board, a), 0) - Qyellow.get(game.boardTup(sOld, aOld), 0))

                # Shift current board and action to old ones.

                sOld, aOld = game.board, a

                # Apply action to get new board.

                game.insert(a, player)

                stepCount += 1

        if game.checkForWin():

            # Update Qold with TDerror (1-Qold)

            if player == 'R':

                numTimesRWon += 1

                Qred[game.boardTup(sOld, aOld)] = 1 + Qred.get(game.boardTup(sOld, aOld), 0)

            if player == 'Y':

                 numTimesYWon += 1

                 Qyellow[game.boardTup(sOld, aOld)] = 1 + Qyellow.get(game.boardTup(sOld, aOld), 0)

            steps.append(stepCount)

    print("R won: ", numTimesRWon)

    print("Y won: ", numTimesYWon)

    return Qred, Qyellow, steps