ニューラルネットワークを使い三目並べゲームを作成してみよう！！

今回はニューラルネットワークを用いて三目並べゲームを作っていきます。参考にしたのは以下のソースでーす。
GitHub

ニューラルネットワークを用いて学習させることにより先手に対して最善の手を導きだします。三目並べは確定ゲームなので
先手は下手なことをしなければ勝ちか引き分けることができます。今回は学習し最善手を見つけ引き分けに持ち込む
プログラミングです。

では作成手順を紹介します
①座標変換をし、最善手を導き出す関数の作成。3×3の升目に対し、左上から右に0～2,3～5,6～8の値がデフォルトであった場合、
90度、180度、270度に回転、もしくは水平方向、垂直方向に反転させた場合の並びを導き出しています。

def get_symmetry(board, response, transformation):
    if transformation == 'rotate180':
        new_response = 8 - response
        return (board[::-1], new_response)
    elif transformation == 'rotate90':
        new_response = [6, 3, 0, 7, 4, 1, 8, 5, 2].index(response)
        tuple_board = list(zip(*[board[6:9], board[3:6], board[0:3]]))
        return ([value for item in tuple_board for value in item], new_response)
    elif transformation == 'rotate270':
        new_response = [2, 5, 8, 1, 4, 7, 0, 3, 6].index(response)
        tuple_board = list(zip(*[board[0:3], board[3:6], board[6:9]]))[::-1]
        return ([value for item in tuple_board for value in item], new_response)
    elif transformation == 'flip_v':
        new_response = [6, 7, 8, 3, 4, 5, 0, 1, 2].index(response)
        return (board[6:9] + board[3:6] + board[0:3], new_response)
    elif transformation == 'flip_h':  # flip_h = rotate180, then flip_v
        new_response = [2, 1, 0, 5, 4, 3, 8, 7, 6].index(response)
        new_board = board[::-1]
        return (new_board[6:9] + new_board[3:6] + new_board[0:3], new_response)
    else:
        raise ValueError('Method not implemented.')

def get_symmetry(board, response, transformation):

if transformation == 'rotate180':

new_response = 8 - response

return (board[::-1], new_response)

elif transformation == 'rotate90':

new_response = [6, 3, 0, 7, 4, 1, 8, 5, 2].index(response)

tuple_board = list(zip(*[board[6:9], board[3:6], board[0:3]]))

return ([value for item in tuple_board for value in item], new_response)

elif transformation == 'rotate270':

new_response = [2, 5, 8, 1, 4, 7, 0, 3, 6].index(response)

tuple_board = list(zip(*[board[0:3], board[3:6], board[6:9]]))[::-1]

return ([value for item in tuple_board for value in item], new_response)

elif transformation == 'flip_v':

new_response = [6, 7, 8, 3, 4, 5, 0, 1, 2].index(response)

return (board[6:9] + board[3:6] + board[0:3], new_response)

elif transformation == 'flip_h': # flip_h = rotate180, then flip_v

new_response = [2, 1, 0, 5, 4, 3, 8, 7, 6].index(response)

new_board = board[::-1]

return (new_board[6:9] + new_board[3:6] + new_board[0:3], new_response)

else:

raise ValueError('Method not implemented.')

②最善手のリストから局面局面における最善手を導き出す関数、ランダムに変換された局面で最善手を導き出す関数を作成

def get_moves_from_csv(csv_file):
    moves = []
    with open(csv_file, 'rt') as csvfile:
        reader = csv.reader(csvfile, delimiter=',')
        for row in reader:
            moves.append(([int(x) for x in row[0:9]], int(row[9])))
    return (moves)



def get_rand_move(moves, n=1, rand_transforms=2):
    (board, response) = random.choice(moves)
    possible_transforms = ['rotate90', 'rotate180', 'rotate270', 'flip_v', 'flip_h']
    for i in range(rand_transforms):
        random_transform = random.choice(possible_transforms)
        (board, response) = get_symmetry(board, response, random_transform)
    return (board, response)

def get_moves_from_csv(csv_file):

moves = []

with open(csv_file, 'rt') as csvfile:

reader = csv.reader(csvfile, delimiter=',')

for row in reader:

moves.append(([int(x) for x in row[0:9]], int(row[9])))

return (moves)

def get_rand_move(moves, n=1, rand_transforms=2):

(board, response) = random.choice(moves)

possible_transforms = ['rotate90', 'rotate180', 'rotate270', 'flip_v', 'flip_h']

for i in range(rand_transforms):

random_transform = random.choice(possible_transforms)

(board, response) = get_symmetry(board, response, random_transform)

return (board, response)

③上記関数を用いててレーニングを開始

X = tf.placeholder(dtype=tf.float32, shape=[None, 9])
Y = tf.placeholder(dtype=tf.int32, shape=[None])

A1 = init_weights([9, 81])
bias1 = init_weights([81])
A2 = init_weights([81, 9])
bias2 = init_weights([9])

model_output = model(X, A1, A2, bias1, bias2)

loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=model_output, labels=Y))
train_step = tf.train.GradientDescentOptimizer(0.025).minimize(loss)
prediction = tf.argmax(model_output, 1)

sess = tf.Session()
init = tf.global_variables_initializer()
sess.run(init)

loss_vec = []
for i in range(10000):
    rand_indices = np.random.choice(range(len(train_set)), batch_size, replace=False)
    batch_data = [train_set[i] for i in rand_indices]
    x_input = [x[0] for x in batch_data]
    y_target = np.array([y[1] for y in batch_data])
    sess.run(train_step, feed_dict={X: x_input, Y: y_target})

    temp_loss = sess.run(loss, feed_dict={X: x_input, Y: y_target})
    loss_vec.append(temp_loss)
    if i % 500 == 0:
        print('iteration ' + str(i) + ' Loss: ' + str(temp_loss))

X = tf.placeholder(dtype=tf.float32, shape=[None, 9])

Y = tf.placeholder(dtype=tf.int32, shape=[None])

A1 = init_weights([9, 81])

bias1 = init_weights([81])

A2 = init_weights([81, 9])

bias2 = init_weights([9])

model_output = model(X, A1, A2, bias1, bias2)

loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=model_output, labels=Y))

train_step = tf.train.GradientDescentOptimizer(0.025).minimize(loss)

prediction = tf.argmax(model_output, 1)

sess = tf.Session()

init = tf.global_variables_initializer()

sess.run(init)

loss_vec = []

for i in range(10000):

rand_indices = np.random.choice(range(len(train_set)), batch_size, replace=False)

batch_data = [train_set[i] for i in rand_indices]

x_input = [x[0] for x in batch_data]

y_target = np.array([y[1] for y in batch_data])

sess.run(train_step, feed_dict={X: x_input, Y: y_target})

temp_loss = sess.run(loss, feed_dict={X: x_input, Y: y_target})

loss_vec.append(temp_loss)

if i % 500 == 0:

print('iteration ' + str(i) + ' Loss: ' + str(temp_loss))

ここまでで三目並べゲームの最善手を導き出きだせる状況ができた。実際使って遊びながら確認してみよう
④勝つか升いっぱいに打ち終わるまで繰り返すプログラミング

game_tracker = [0., 0., 0., 0., 0., 0., 0., 0., 0.]
win_logical = False
num_moves = 0
while not win_logical:
    player_index = input('Input index of your move (0-8): ')
    num_moves += 1
    game_tracker[int(player_index)] = 1.

    [potential_moves] = sess.run(model_output, feed_dict={X: [game_tracker]})
    allowed_moves = [ix for ix, x in enumerate(game_tracker) if x == 0.0]
    model_move = np.argmax([x if ix in allowed_moves else -999.0 for ix, x in enumerate(potential_moves)])

    game_tracker[int(model_move)] = -1.
    print('Model has moved')
    print_board(game_tracker)
    if check(game_tracker) == 1 or num_moves &gt;= 5:
        print('Game Over!')
        win_logical = True

game_tracker = [0., 0., 0., 0., 0., 0., 0., 0., 0.]

win_logical = False

num_moves = 0

while not win_logical:

player_index = input('Input index of your move (0-8): ')

num_moves += 1

game_tracker[int(player_index)] = 1.

[potential_moves] = sess.run(model_output, feed_dict={X: [game_tracker]})

allowed_moves = [ix for ix, x in enumerate(game_tracker) if x == 0.0]

model_move = np.argmax([x if ix in allowed_moves else -999.0 for ix, x in enumerate(potential_moves)])

game_tracker[int(model_move)] = -1.

print('Model has moved')

print_board(game_tracker)

if check(game_tracker) == 1 or num_moves >= 5:

print('Game Over!')

win_logical = True

実際動かしてみるとおおよそこちらの手に対して最善の手を打ってきているように感じた。この内容をさらに突き詰めていくとAlphaGOの
ようなAIができるのかなと感じました。まだまだ勉強が必要ですね。