%% LyX 2.3.7 created this file.  For more info, see http://www.lyx.org/.
%% Do not edit unless you really know what you are doing.
\documentclass[english]{article}
\usepackage[T1]{fontenc}
\usepackage[latin9]{inputenc}
\usepackage{geometry}
\geometry{verbose,tmargin=2cm,bmargin=2cm,lmargin=3cm,rmargin=3cm}
\usepackage{amsmath}
\usepackage{amsthm}

\makeatletter
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% Textclass specific LaTeX commands.
\theoremstyle{plain}
\newtheorem{thm}{\protect\theoremname}
\theoremstyle{definition}
\newtheorem{xca}[thm]{\protect\exercisename}
\theoremstyle{definition}
\newtheorem{example}[thm]{\protect\examplename}

\@ifundefined{date}{}{\date{}}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% User specified LaTeX commands.

\usepackage{babel}
\usepackage{color}
\usepackage{xcolor}
\usepackage[normalem]{ulem}
\usepackage{amsfonts}
\makeatletter
\DeclareFontFamily{U}{tipa}{}
\DeclareFontShape{U}{tipa}{m}{n}{<->tipa10}{}
\newcommand{\arc@char}{{\usefont{U}{tipa}{m}{n}\symbol{62}}}%

\newcommand{\arc}[1]{\mathpalette\arc@arc{#1}}

\newcommand{\arc@arc}[2]{%
  \sbox0{$\m@th#1#2$}%
  \vbox{
    \hbox{\resizebox{\wd0}{\height}{\arc@char}}
    \nointerlineskip
    \box0
  }%
}
\makeatother

\makeatother

\usepackage{babel}
\providecommand{\examplename}{Example}
\providecommand{\exercisename}{Exercise}
\providecommand{\theoremname}{Theorem}

\begin{document}
\title{Exercise for Optimal control -- Week 6}
\author{Choose \textbf{1.5 }problems to solve.}
\maketitle
\begin{xca}
Derive the policy iteration scheme for the LQR problem 
\[
\min_{u(\cdot)}\sum_{k=1}^{\infty}x_{k}^{\top}Qx_{k}+u_{k}^{\top}Ru_{k}
\]
with $Q=Q^{\top}\ge0$ and $R=R^{\top}>0$ subject to:
\[
x_{k+1}=Ax_{k}+Bu_{k}.
\]
Assume the system is stabilizable. Start the iteration with a stabilizing
policy. Run the policy iteration and value iteration on a computer
for the following matrices:
\[
A=\begin{bmatrix}0 & 1 & 0\\
0 & 0 & 1\\
1 & -1 & 2
\end{bmatrix},\quad b=\begin{bmatrix}0\\
0\\
1
\end{bmatrix},\quad Q=\begin{bmatrix}0 & 0 & 0\\
0 & 0 & 0\\
0 & 0 & 1
\end{bmatrix},\quad R=I
\]
Compare the convergence rates of the two iterations scheme for the
policies and value functions.
\end{xca}

\begin{xca}[LQR for LTV systems]
Consider a controllable LTV system 
\begin{align*}
\dot{x} & =A(t)x+B(t)u
\end{align*}
with $x\in\mathbb{R}^{n}$, $u\in\mathbb{R}^{m}$ and cost function
\[
J=x(T)^{\top}Q_{f}x(T)+\int_{t_{0}}^{T}x(t)^{\top}Q(t)x(t)+u(t)^{\top}R(t)u(t){\rm d}t
\]
where $Q_{f},\,Q(t)\ge0$ and $R(t)>0$ for all $t\ge0$. In addition,
we assume $A(\cdot)$, $B(\cdot)$, $Q(\cdot)$ and $R(\cdot)$ are
continuous. The objective is to find an optimal control $u^{*}$ such
that $J$ is minimized. 

1) The dynamic programming works also for time varying systems. Write
down the Hamiltonian $H(t,x,u,p)$ for this problem and derive the
optimal controller using the verification rule. \emph{Hint: }consider
value function of the form $J^{*}(t,x)=x^{\top}P(t)x$.

2) Show that the HJB equation reduces to an ODE:
\begin{equation}
-\dot{P}(t)=Q(t)+P(t)A(t)+A(t)^{\top}P(t)-P(t)B(t)R(t)^{-1}B(t)^{\top}P(t).\label{eq:DRE}
\end{equation}
with boundary condition
\[
P(T)=Q_{f}.
\]

3) Prove that the equation \eqref{eq:DRE} has a unique symmetric
semi-positive definite solution on interval $[0,T]$ for any $T>0$.
In particular, there is no finite escape time.
\end{xca}

\begin{example}
1) Derive the HJB equation for the time optimal control problem of
the double integrator
\begin{align*}
\dot{x}_{1} & =x_{2}\\
\dot{x}_{2} & =u
\end{align*}
with initial condition $(1,1)$ and terminal condition $(0,0)$ under
the constraint $|u|\le1$.

2) Solve the HJB equation using method of characteristics.
\end{example}


\end{document}
