A Linear Quadratic Regulator `Task`.
| 209 | |
| 210 | |
| 211 | class LQRLevel(base.Task): |
| 212 | """A Linear Quadratic Regulator `Task`.""" |
| 213 | |
| 214 | _TERMINAL_TOL = 1e-6 |
| 215 | |
| 216 | def __init__(self, control_cost_coef, random=None): |
| 217 | """Initializes an LQR level with cost = sum(states^2) + c*sum(controls^2). |
| 218 | |
| 219 | Args: |
| 220 | control_cost_coef: The coefficient of the control cost. |
| 221 | random: Optional, either a `numpy.random.RandomState` instance, an |
| 222 | integer seed for creating a new `RandomState`, or None to select a seed |
| 223 | automatically (default). |
| 224 | |
| 225 | Raises: |
| 226 | ValueError: If the control cost coefficient is not positive. |
| 227 | """ |
| 228 | if control_cost_coef <= 0: |
| 229 | raise ValueError('control_cost_coef must be positive.') |
| 230 | |
| 231 | self._control_cost_coef = control_cost_coef |
| 232 | super().__init__(random=random) |
| 233 | |
| 234 | @property |
| 235 | def control_cost_coef(self): |
| 236 | return self._control_cost_coef |
| 237 | |
| 238 | def initialize_episode(self, physics): |
| 239 | """Random state sampled from a unit sphere.""" |
| 240 | ndof = physics.model.nq |
| 241 | unit = self.random.randn(ndof) |
| 242 | physics.data.qpos[:] = np.sqrt(2) * unit / np.linalg.norm(unit) |
| 243 | super().initialize_episode(physics) |
| 244 | |
| 245 | def get_observation(self, physics): |
| 246 | """Returns an observation of the state.""" |
| 247 | obs = collections.OrderedDict() |
| 248 | obs['position'] = physics.position() |
| 249 | obs['velocity'] = physics.velocity() |
| 250 | return obs |
| 251 | |
| 252 | def get_reward(self, physics): |
| 253 | """Returns a quadratic state and control reward.""" |
| 254 | position = physics.position() |
| 255 | state_cost = 0.5 * np.dot(position, position) |
| 256 | control_signal = physics.control() |
| 257 | control_l2_norm = 0.5 * np.dot(control_signal, control_signal) |
| 258 | return 1 - (state_cost + control_l2_norm * self._control_cost_coef) |
| 259 | |
| 260 | def get_evaluation(self, physics): |
| 261 | """Returns a sparse evaluation reward that is not used for learning.""" |
| 262 | return float(physics.state_norm() <= 0.01) |
| 263 | |
| 264 | def get_termination(self, physics): |
| 265 | """Terminates when the state norm is smaller than epsilon.""" |
| 266 | if physics.state_norm() < self._TERMINAL_TOL: |
| 267 | return 0.0 |
no outgoing calls
no test coverage detected
searching dependent graphs…