gee

""" Trains an agent with (stochastic) Policy Gradients on Pong. Uses OpenAI Gym. """ import numpy as np import cPickle as pickle import gym

hyperparameters

H = 200 # number of hidden layer neurons batch_size = 10 # every how many episodes to do a param update? 10판이 끝나고 학습량 업데이트. learning_rate = 1e-4 gamma = 0.99 # discount factor for reward decay_rate = 0.99 # decay factor for RMSProp leaky sum of grad^2 resume = False # resume from previous checkpoint? render = False

model initialization

D = 80 * 80 # input dimensionality: 80x80 grid if resume: # resume option인 경우 세이브파일을 읽어온다. (자유로운 형식인 피클파일) model = pickle.load(open('save.p', 'rb')) # 바이너리 형식의 세이브파일을 읽는다. else: model = {} #딕셔너리 타입의 모델을 정의 # Return a sample (or samples) from the “standard normal” distribution. np.random.randn # 내부 파라미터의 수에 따라 dimension 결정. 여기서는 1개 2개. model['W1'] = np.random.randn(H, D) / np.sqrt(D) # "Xavier" initialization (모르겠다) model['W2'] = np.random.randn(H) / np.sqrt(H)

dictionary 루프문

k,v 를 둘다 포함하는 iterator이다.

딕셔너리의 밸류 배열 내부를 0으로 초기화

grad_buffer = {k: np.zeros_like(v) for k, v in model.iteritems()} # update buffers that add up gradients over a batch rmsprop_cache = {k: np.zeros_like(v) for k, v in model.iteritems()} # rmsprop memory

그냥 심플한 sigmoid

def sigmoid(x): return 1.0 / (1.0 + np.exp(-x)) # sigmoid "squashing" function to interval [0,1]

def prepro(I): """ prepro 210x160x3 uint8 frame into 6400 (80x80) 1D float vector """ I = I[35:195] # crop # ::2(double colon) means nothing for first,second argument and jump to 2. (factor of 2) I = I[::2, ::2, 0] # downsample by factor of 2 # false를 리턴해서 첫원소가 0으로 바뀐다는건가..?.. I[I == 144] = 0 # erase background (background type 1) I[I == 109] = 0 # erase background (background type 2) # true를 리턴하여 2번째원소만 1로 ? I[I != 0] = 1 # everything else (paddles, ball) just set to 1 # I의 내부원소타입을 float로 하고 ravel - flattend 1D어레이를 리턴한다. (맨위 영어주석에 나온대로) return I.astype(np.float).ravel()

def discount_rewards(r): """ take 1D float array of rewards and compute discounted reward """ discounted_r = np.zeros_like(r) running_add = 0 # reversed를 통해 역방향 iterator 생성 for t in reversed(xrange(0, r.size)): if r[t] != 0: running_add = 0 # reset the sum, since this was a game boundary (pong specific!) # 나머지 경우에 대해 디스카운트 리워드 적용해준다. running_add = running_add * gamma + r[t] discounted_r[t] = running_add return discounted_r

up and down 액션의 확률을 구한다.

def policy_forward(x): # h에 내적값 넣는다. # h: 입력값과 히든 유닛의 중간값. h = np.dot(model['W1'], x) #RElu nonlinearity?? # recitifier function. _/ h[h < 0] = 0 # ReLU nonlinearity # 출력레이어 logp = np.dot(model['W2'], h) #활성화함수 - sigmoid p = sigmoid(logp) return p, h # return probability of taking action 2, and hidden state

def policy_backward(eph, epdlogp): """ backward pass. (eph is array of intermediate hidden states) """ dW2 = np.dot(eph.T, epdlogp).ravel() dh = np.outer(epdlogp, model['W2']) dh[eph <= 0] = 0 # backpro prelu dW1 = np.dot(dh.T, epx) return {'W1': dW1, 'W2': dW2}

env = gym.make("Pong-v0") observation = env.reset() prev_x = None # used in computing the difference frame xs, hs, dlogps, drs = [], [], [], [] running_reward = None reward_sum = 0 episode_number = 0