定义
$$
X = \begin{pmatrix}
{x_{1}}&{x_{2}}&{\cdots}&{x_{N}}
\end{pmatrix}^T_{N\times p}
$$ $$
Y=\left(\begin{array}{l}
{y_{1}} & {y_{2}} & {\cdots} & {y_{N}}
\end{array}\right)^T_{N \times 1}
$$
\begin{array}{l}
{\left\lbrace \left(x_{i},\quad y_{i}\right)\right\rbrace_{i=1}^{N},\quad x_{i} \in \mathbb{R}^{p},\quad y_{i} \in\lbrace+1,-1\rbrace} \\
{x_{c_{1}}=\left\lbrace x_{i} | y_{i}=+1\right\rbrace, \quad x_{c_{2}}=\left\lbrace x_{i} | y_{i}=-1\right\rbrace} \\
{\left|x_{c_{1}}\right|=N_{1}, \quad\left|x_{c_{2}}\right|=N_{2}, \quad N_{1}+N_{2}=N}
\end{array}
思想
线性判别分析的思想是使得类内差异小，类间差异大。
$$\begin{aligned} z_{i} &=w^{\top} x_{i} \\ \bar{z} &=\frac{1}{N} \sum_{i=1}^{N} z_{i}=\frac{1}{N} \sum_{i=1}^{N} w^{\top} x_{i} \\ S &=\frac{1}{N} \sum_{i=1}^{N}\left(z_{i}-\bar{z}\right)\left(z_{i}-\bar{z}\right)^{\top} \\ &=\frac{1}{N} \sum_{i=1}^{N}\left(w^{\top} x_{i}-\bar{z}\right)\left(w^{\top} x_{i}-\bar{z}\right)^{\top} \end{aligned}$$
$$\begin{aligned}
C_1: \bar{z}{1} &=\frac{1}{N_1} \sum{i=1}^{N_1} w^{\top} x_i \\
S_{1} &=\frac{1}{N_{1}} \sum_{i=1}^{N_1}\left(w^{\top} x_i-\bar{z}_1 \right)\left(w^{\top} x_i-\bar{z}_1 \right)^{\top} \\
\end{aligned}$$
$$\begin{aligned}
C_2: \bar{z}{2} &=\frac{1}{N_2} \sum{i=1}^{N_2} w^{\top} x_i \\
S_{2} &=\frac{1}{N_{2}} \sum_{i=1}^{N_2}\left(w^{\top} x_i-\bar{z}_2 \right)\left(w^{\top} x_i-\bar{z}_2 \right)^{\top}
\end{aligned}$$
目标函数
$$J(w) = \frac{\left( \bar{z}_{1} - \bar{z}_2 \right)^2 } {s_1+s_2}$$
$$\hat{w}=\arg\max_{w} J(w)$$
分子
$$\begin{aligned}
\left(\bar{z}{1} - \bar{z}2 \right)^2 &= \left(\frac{1}{N_1} \sum{i=1}^{N{1}} w^{\top} x_{i}-\frac{1}{N_{2}} \sum_{i=1}^{N_{2}} w^{\top} x_{i}\right)^2=\left[w^{\top}\left(\frac{1}{N_{1}} \sum_{i=1}^{N_{2}} x_{i}-\frac{1}{N_{2}} \sum_{i=1}^{N_{2}} x_{i}\right)\right]^2 \\
&= \left(w^{\top}\left(\bar{x}{c_1}-\bar{x}{c_2}\right)\right)^{2}=w^{\top}\left(\bar{x}{c_1}-\bar{x}{c_{2}}\right)\left(\bar{x}{c{1}}-\bar{x}_{c_2}\right)^{\top} w
\end{aligned}
$$
分别求分母 $S_1$ 和 $S_2$
\begin{aligned}
S_{1} &=\frac{1}{N_{1}} \sum_{i=1}^{N_{1}}\left(w^{\top} x_{i}-\frac{1}{N_{1}} \sum_{j=1}^{N_{1}} w^{\top} x_{j}\right)\left(w^{\top} x_{i}-\frac{1}{N_{1}} \sum_{j=1}^{N_{1}} w^{\top} x_{j}\right)^{\top} \\
&=\frac{1}{N_{1}} \sum_{i=1}^{N_{1}} w^{\top}\left(x_{i}-\bar{x}{c{1}}\right)\left(x_{i}-\bar{x}{c_1}\right)^{\top} w \\
&=w^{\top}\left[\frac{1}{N{1}} \sum_{i=1}^{N}\left(x_{i}-\bar{x}{c_1}\right)\left(x{i}-\bar{x}{c_1}\right)^{\top}\right] w \\
&=w^{\top} \quad \cdot \quad S{c_1} \quad \cdot w \\
&=w^{\top} S_{c_1} w
\end{aligned}
分母
\begin{aligned}
S_1 + S_2 &=w^{\top} S_{c_1} w+w^{\top} S_{c_2} w\\
&=w^{T}\left(S_{c_1}+S_{c_{2}}\right) w
\end{aligned}
$$
J(w)=\frac{w^{\top}\left(\bar{x}{c_1}-\bar{x}{c_{2}}\right)\left(\bar{x}{c{1}}-\bar{x}{c{2}}\right)^{\top} w}{w^{\top}\left(s_{c_{1}}+s_{c_{2}}\right) w}
$$
模型求解
\begin{aligned}
J(w) &= \frac{\left( \bar{z}{1} - \bar{z}2 \right)^2 } {s_1+s_2} \\
&=\frac{w^{\top}\left(\bar{x}{c_1}-\bar{x}{c_2}\right)\left(\bar{x}{c_1}-\bar{x}{c_2}\right)^{\top} w}{w^{\top}\left(s_{c_1}+s_{c_2}\right) w} \\
&=\frac{w^{\top} S_{b} w}{w^{\top} S_w w}
\end{aligned}
\begin{aligned}
&S_b=\left(\bar{x}{c_1}-\bar{x}{c_2} \right) \left(\bar{x}{c_1}-\bar{x}{c_2}\right)^{\top}\\
&S_w=S_{c_1}+S_{c_2}
\end{aligned}
$ S_b $ : between-class 类间方差 $p\times p$
$ S_w $ : within-class 类内方差 $p\times p$
对$J(w)$求导，并令其为$0$，
\begin{aligned}
\frac{\partial J(w)}{\partial w} &=2 S_{b} w\left(w^{\top} S_{w} w\right)^{-1}+w^{\top} S_{b} w \cdot(-1) \cdot\left(w^{\top} S_{w} w\right)^{-2} \\
&=0
\end{aligned}
\begin{aligned}
\hat{w} &=\frac{w^{\top} S w}{w^{\top} S_{b} w} S_{w}^{-1} \cdot S_{b} \cdot w\\
&\propto S_w^{-1} \cdot S_{b} \cdot w \\
&= S_w^{-1} \cdot (\bar{x}{c_1} - \bar{x}{c_2})(\bar{x}{c_1} - \bar{x}{c_2})^{\top} \cdot w \\
&\propto S_w^{-1} \cdot (\bar{x}{c_1} - \bar{x}{c_2})
\end{aligned}
