cartpole.py文件下
類成員變量的定義
def __init__(self):
self.gravity = 9.8
self.masscart = 1.0 #車子的質量
self.masspole = 0.1 #杆的質量
self.total_mass = (self.masspole + self.masscart) #總質量
self.length = 0.5 # actually half the pole's length實際杆長的一半
self.polemass_length = (self.masspole * self.length)
self.force_mag = 10.0 // 我理解這裏是系統所受的力
self.tau = 0.02 # seconds between state updates // 兩幀之間更新的時間
self.kinematics_integrator = 'euler'
# Angle at which to fail the episode
self.theta_threshold_radians = 12 * 2 * math.pi / 360 //角度的閾值
self.x_threshold = 2.4
# Angle limit set to 2 * theta_threshold_radians so failing observation is still within bounds
high = np.array([
self.x_threshold * 2,
np.finfo(np.float32).max,
self.theta_threshold_radians * 2,
np.finfo(np.float32).max])
self.action_space = spaces.Discrete(2)#動作空間，目前就兩個動作
self.observation_space = spaces.Box(-high, high, dtype=np.float32)#觀測矩陣
self.seed()
self.viewer = None
self.state = None
self.steps_beyond_done = None
step函數：
def step(self, action):
assert self.action_space.contains(action), "%r (%s) invalid"%(action, type(action))
state = self.state
x, x_dot, theta, theta_dot = state #theta是角度，theta_dot是轉速
force = self.force_mag if action==1 else -self.force_mag #個人理解這代表系統水平方向上的受力
costheta = math.cos(theta)
sintheta = math.sin(theta)
temp = (force + self.polemass_length * theta_dot * theta_dot * sintheta) / self.total_mass ##車擺的動力學方程式，即加速度與動作之間的關係。
thetaacc = (self.gravity * sintheta - costheta* temp) / (self.length * (4.0/3.0 - self.masspole * costheta * costheta / self.total_mass)) #擺的角加速度
xacc = temp - self.polemass_length * thetaacc * costheta / self.total_mass #小車的平移加速
if self.kinematics_integrator == 'euler':
x = x + self.tau * x_dot
x_dot = x_dot + self.tau * xacc
theta = theta + self.tau * theta_dot
theta_dot = theta_dot + self.tau * thetaacc
else: # semi-implicit euler
x_dot = x_dot + self.tau * xacc
x = x + self.tau * x_dot
theta_dot = theta_dot + self.tau * thetaacc
theta = theta + self.tau * theta_dot
self.state = (x,x_dot,theta,theta_dot)
done = x < -self.x_threshold \
or x > self.x_threshold \
or theta < -self.theta_threshold_radians \
or theta > self.theta_threshold_radians
done = bool(done)
if not done:
reward = 1.0
elif self.steps_beyond_done is None:
# Pole just fell!
self.steps_beyond_done = 0
reward = 1.0
else:
if self.steps_beyond_done == 0:
logger.warn("You are calling 'step()' even though this environment has already returned done = True. You should always call 'reset()' once you receive 'done = True' -- any further steps are undefined behavior.")
self.steps_beyond_done += 1
reward = 0.0
return np.array(self.state), reward, done, {}