Machine Learning & Signals Learning

$\newcommand{\footnotename}{footnote}$ $\def \LWRfootnote {1}$ $\newcommand {\footnote }[2][\LWRfootnote ]{{}^{\mathrm {#1}}}$ $\newcommand {\footnotemark }[1][\LWRfootnote ]{{}^{\mathrm {#1}}}$ $\let \LWRorighspace \hspace $ $\renewcommand {\hspace }{\ifstar \LWRorighspace \LWRorighspace }$ $\newcommand {\TextOrMath }[2]{#2}$ $\newcommand {\mathnormal }[1]{{#1}}$ $\newcommand \ensuremath [1]{#1}$ $\newcommand {\LWRframebox }[2][]{\fbox {#2}} \newcommand {\framebox }[1][]{\LWRframebox } $ $\newcommand {\setlength }[2]{}$ $\newcommand {\addtolength }[2]{}$ $\newcommand {\setcounter }[2]{}$ $\newcommand {\addtocounter }[2]{}$ $\newcommand {\arabic }[1]{}$ $\newcommand {\number }[1]{}$ $\newcommand {\noalign }[1]{\text {#1}\notag \\}$ $\newcommand {\cline }[1]{}$ $\newcommand {\directlua }[1]{\text {(directlua)}}$ $\newcommand {\luatexdirectlua }[1]{\text {(directlua)}}$ $\newcommand {\protect }{}$ $\def \LWRabsorbnumber #1 {}$ $\def \LWRabsorbquotenumber "#1 {}$ $\newcommand {\LWRabsorboption }[1][]{}$ $\newcommand {\LWRabsorbtwooptions }[1][]{\LWRabsorboption }$ $\def \mathchar {\ifnextchar "\LWRabsorbquotenumber \LWRabsorbnumber }$ $\def \mathcode #1={\mathchar }$ $\let \delcode \mathcode $ $\let \delimiter \mathchar $ $\def \oe {\unicode {x0153}}$ $\def \OE {\unicode {x0152}}$ $\def \ae {\unicode {x00E6}}$ $\def \AE {\unicode {x00C6}}$ $\def \aa {\unicode {x00E5}}$ $\def \AA {\unicode {x00C5}}$ $\def \o {\unicode {x00F8}}$ $\def \O {\unicode {x00D8}}$ $\def \l {\unicode {x0142}}$ $\def \L {\unicode {x0141}}$ $\def \ss {\unicode {x00DF}}$ $\def \SS {\unicode {x1E9E}}$ $\def \dag {\unicode {x2020}}$ $\def \ddag {\unicode {x2021}}$ $\def \P {\unicode {x00B6}}$ $\def \copyright {\unicode {x00A9}}$ $\def \pounds {\unicode {x00A3}}$ $\let \LWRref \ref $ $\renewcommand {\ref }{\ifstar \LWRref \LWRref }$ $ \newcommand {\multicolumn }[3]{#3}$ $\require {textcomp}$ $ \newcommand {\abs }[1]{\lvert #1\rvert } $ $ \DeclareMathOperator {\sign }{sign} $ $\newcommand {\intertext }[1]{\text {#1}\notag \\}$ $\let \Hat \hat $ $\let \Check \check $ $\let \Tilde \tilde $ $\let \Acute \acute $ $\let \Grave \grave $ $\let \Dot \dot $ $\let \Ddot \ddot $ $\let \Breve \breve $ $\let \Bar \bar $ $\let \Vec \vec $ $\newcommand {\bm }[1]{\boldsymbol {#1}}$ $\require {physics}$ $\newcommand {\LWRphystrig }[2]{\ifblank {#1}{\textrm {#2}}{\textrm {#2}^{#1}}}$ $\renewcommand {\sin }[1][]{\LWRphystrig {#1}{sin}}$ $\renewcommand {\sinh }[1][]{\LWRphystrig {#1}{sinh}}$ $\renewcommand {\arcsin }[1][]{\LWRphystrig {#1}{arcsin}}$ $\renewcommand {\asin }[1][]{\LWRphystrig {#1}{asin}}$ $\renewcommand {\cos }[1][]{\LWRphystrig {#1}{cos}}$ $\renewcommand {\cosh }[1][]{\LWRphystrig {#1}{cosh}}$ $\renewcommand {\arccos }[1][]{\LWRphystrig {#1}{arcos}}$ $\renewcommand {\acos }[1][]{\LWRphystrig {#1}{acos}}$ $\renewcommand {\tan }[1][]{\LWRphystrig {#1}{tan}}$ $\renewcommand {\tanh }[1][]{\LWRphystrig {#1}{tanh}}$ $\renewcommand {\arctan }[1][]{\LWRphystrig {#1}{arctan}}$ $\renewcommand {\atan }[1][]{\LWRphystrig {#1}{atan}}$ $\renewcommand {\csc }[1][]{\LWRphystrig {#1}{csc}}$ $\renewcommand {\csch }[1][]{\LWRphystrig {#1}{csch}}$ $\renewcommand {\arccsc }[1][]{\LWRphystrig {#1}{arccsc}}$ $\renewcommand {\acsc }[1][]{\LWRphystrig {#1}{acsc}}$ $\renewcommand {\sec }[1][]{\LWRphystrig {#1}{sec}}$ $\renewcommand {\sech }[1][]{\LWRphystrig {#1}{sech}}$ $\renewcommand {\arcsec }[1][]{\LWRphystrig {#1}{arcsec}}$ $\renewcommand {\asec }[1][]{\LWRphystrig {#1}{asec}}$ $\renewcommand {\cot }[1][]{\LWRphystrig {#1}{cot}}$ $\renewcommand {\coth }[1][]{\LWRphystrig {#1}{coth}}$ $\renewcommand {\arccot }[1][]{\LWRphystrig {#1}{arccot}}$ $\renewcommand {\acot }[1][]{\LWRphystrig {#1}{acot}}$ $\require {cancel}$ $\newcommand *{\underuparrow }[1]{{\underset {\uparrow }{#1}}}$ $\DeclareMathOperator *{\argmax }{argmax}$ $\DeclareMathOperator *{\argmin }{arg\,min}$ $\def \E [#1]{\mathbb {E}\!\left [ #1 \right ]}$ $\def \Var [#1]{\operatorname {Var}\!\left [ #1 \right ]}$ $\def \Cov [#1]{\operatorname {Cov}\!\left [ #1 \right ]}$ $\newcommand {\floor }[1]{\lfloor #1 \rfloor }$ $\newcommand {\DTFTH }{ H \brk 1{e^{j\omega }}}$ $\newcommand {\DTFTX }{ X\brk 1{e^{j\omega }}}$ $\newcommand {\DFTtr }[1]{\mathrm {DFT}\left \{#1\right \}}$ $\newcommand {\DTFTtr }[1]{\mathrm {DTFT}\left \{#1\right \}}$ $\newcommand {\DTFTtrI }[1]{\mathrm {DTFT^{-1}}\left \{#1\right \}}$ $\newcommand {\Ftr }[1]{ \mathcal {F}\left \{#1\right \}}$ $\newcommand {\FtrI }[1]{ \mathcal {F}^{-1}\left \{#1\right \}}$ $\newcommand {\Zover }{\overset {\mathscr Z}{\Longleftrightarrow }}$ $\renewcommand {\real }{\mathbb {R}}$ $\newcommand {\ba }{\mathbf {a}}$ $\newcommand {\bb }{\mathbf {b}}$ $\newcommand {\bd }{\mathbf {d}}$ $\newcommand {\be }{\mathbf {e}}$ $\newcommand {\bh }{\mathbf {h}}$ $\newcommand {\bn }{\mathbf {n}}$ $\newcommand {\bq }{\mathbf {q}}$ $\newcommand {\br }{\mathbf {r}}$ $\newcommand {\bt }{\mathbf {t}}$ $\newcommand {\bv }{\mathbf {v}}$ $\newcommand {\bw }{\mathbf {w}}$ $\newcommand {\bx }{\mathbf {x}}$ $\newcommand {\bxx }{\mathbf {xx}}$ $\newcommand {\bxy }{\mathbf {xy}}$ $\newcommand {\by }{\mathbf {y}}$ $\newcommand {\byy }{\mathbf {yy}}$ $\newcommand {\bz }{\mathbf {z}}$ $\newcommand {\bA }{\mathbf {A}}$ $\newcommand {\bB }{\mathbf {B}}$ $\newcommand {\bI }{\mathbf {I}}$ $\newcommand {\bK }{\mathbf {K}}$ $\newcommand {\bP }{\mathbf {P}}$ $\newcommand {\bQ }{\mathbf {Q}}$ $\newcommand {\bR }{\mathbf {R}}$ $\newcommand {\bU }{\mathbf {U}}$ $\newcommand {\bW }{\mathbf {W}}$ $\newcommand {\bX }{\mathbf {X}}$ $\newcommand {\bY }{\mathbf {Y}}$ $\newcommand {\bZ }{\mathbf {Z}}$ $\newcommand {\balpha }{\bm {\alpha }}$ $\newcommand {\bth }{{\bm {\theta }}}$ $\newcommand {\bepsilon }{{\bm {\epsilon }}}$ $\newcommand {\bmu }{{\bm {\mu }}}$ $\newcommand {\bOne }{\mathbf {1}}$ $\newcommand {\bZero }{\mathbf {0}}$ $\newcommand {\loss }{\mathcal {L}}$ $\newcommand {\appropto }{\mathrel {\vcenter { \offinterlineskip \halign {\hfil $##$\cr \propto \cr \noalign {\kern 2pt}\sim \cr \noalign {\kern -2pt}}}}}$ $\newcommand {\SSE }{\mathrm {SSE}}$ $\newcommand {\MSE }{\mathrm {MSE}}$ $\newcommand {\RMSE }{\mathrm {RMSE}}$ $\newcommand {\toprule }[1][]{\hline }$ $\let \midrule \toprule $ $\let \bottomrule \toprule $ $\def \LWRbooktabscmidruleparen (#1)#2{}$ $\newcommand {\LWRbooktabscmidrulenoparen }[1]{}$ $\newcommand {\cmidrule }[1][]{\ifnextchar (\LWRbooktabscmidruleparen \LWRbooktabscmidrulenoparen }$ $\newcommand {\morecmidrules }{}$ $\newcommand {\specialrule }[3]{\hline }$ $\newcommand {\addlinespace }[1][]{}$ $\newcommand {\LWRsubmultirow }[2][]{#2}$ $\newcommand {\LWRmultirow }[2][]{\LWRsubmultirow }$ $\newcommand {\multirow }[2][]{\LWRmultirow }$ $\newcommand {\mrowcell }{}$ $\newcommand {\mcolrowcell }{}$ $\newcommand {\STneed }[1]{}$ $\newcommand {\tcbset }[1]{}$ $\newcommand {\tcbsetforeverylayer }[1]{}$ $\newcommand {\tcbox }[2][]{\boxed {\text {#2}}}$ $\newcommand {\tcboxfit }[2][]{\boxed {#2}}$ $\newcommand {\tcblower }{}$ $\newcommand {\tcbline }{}$ $\newcommand {\tcbtitle }{}$ $\newcommand {\tcbsubtitle [2][]{\mathrm {#2}}}$ $\newcommand {\tcboxmath }[2][]{\boxed {#2}}$ $\newcommand {\tcbhighmath }[2][]{\boxed {#2}}$

Part IV Appendix

A Notation

Numbers and indexing

.
$a$	Scalar
$\ba $	Vector
$a_i$	Element $i$ of a vector $a$, indexing starting at 1
$\mathbf {A}$	Matrix
$a_{ij}$	Element $i,j$ of a matrix $\mathbf {A}$, indexing starting at 1
$\real $	Real numbers domain
$\real ^D$	$D$-dimensional vector
$\real ^{D_1\times D_2}$	matrix of a dimension $D_1\times D_2$
$\bI $	Identity matrix
$\bOne $	Vector/matrix of ones
$\bZero $	Vector/matrix of zeros

Datasets

.
$L$	Model complexity
$N$	Number of features
$M$	Number of entries in the dataset
$K$	Number of classes
$\bw $ or $w_i$	Model parameters (vector form)
$f(\cdot ;\bw )$	Model
$h(\bx )$ or $h(x)$	True unknown function
$x_{ij}$	Singe data value
$\bx _i$	Singe data vector, $i$ column number in $\bX $
$\bX $	Data matrix
$\by $	Target vector for the data in $\bX $
$\hat {\by }$	Prediction vector of $\by $
$y_i$	Target value
$\hat {y}_i$	Predicted target value
$\loss (\by ,\hat {\by })$ or $\mathcal {L}(y_i,\hat {y}_i)$	Loss function
$\lambda $	Regularization parameter
$\ba ^{[k]}$	Activation of layer $k$
$\bz ^{[k]}$	Output of layer $k$
$g_k(\cdot )$	Activation function of layer $k$
$\bth $ or $\theta _i$	Model parameters (general form)
$\balpha $	Kernel/dual coefficients vector
$\be $	Error/residual vector
$\bepsilon $ or $\epsilon _i$	Noise vector/term
$\bn $	Noise vector (signal processing)
$\bh $	Impulse response / filter coefficients
$\bP $	Projection matrix
$\bK $	Kernel matrix
$\bR $	Autocorrelation matrix
$\phi (\cdot )$	Feature mapping / basis function
$\alpha $	Learning rate (gradient descent step size)

Statistics

.
$x$	Sample set
$\bar x$	Sample mean
$s_x^2$	Sample variance (biased or unbiased)
$s_x$	Sample std (biased or unbiased)
$s_{xy}$	Sample covariance (biased or unbiased)
$r_{xy}$	Sample correlation coefficient
$\mu $	Population mean
$\sigma ^2$	Population variance
$\sigma $	Population standard deviation
$\E [\cdot ]$	Expectation operator
$\Var [\cdot ]$	Variance operator
$\Cov [\cdot ]$	Covariance operator

Signals

.
$\omega $	Angular frequency (discrete)
$\theta $	Phase angle
$A$	Amplitude
$F$	Frequency [Hz]
$F_s$	Sampling frequency
$T$	Period [sec]

.
\(a\)	Scalar
\(\ba \)	Vector
\(a_i\)	Element \(i\) of a vector \(a\), indexing starting at 1
\(\mathbf {A}\)	Matrix
\(a_{ij}\)	Element \(i,j\) of a matrix \(\mathbf {A}\), indexing starting at 1
\(\real \)	Real numbers domain
\(\real ^D\)	\(D\)-dimensional vector
\(\real ^{D_1\times D_2}\)	matrix of a dimension \(D_1\times D_2\)
\(\bI \)	Identity matrix
\(\bOne \)	Vector/matrix of ones
\(\bZero \)	Vector/matrix of zeros

.
\(L\)	Model complexity
\(N\)	Number of features
\(M\)	Number of entries in the dataset
\(K\)	Number of classes
\(\bw \) or \(w_i\)	Model parameters (vector form)
\(f(\cdot ;\bw )\)	Model
\(h(\bx )\) or \(h(x)\)	True unknown function
\(x_{ij}\)	Singe data value
\(\bx _i\)	Singe data vector, \(i\) column number in \(\bX \)
\(\bX \)	Data matrix
\(\by \)	Target vector for the data in \(\bX \)
\(\hat {\by }\)	Prediction vector of \(\by \)
\(y_i\)	Target value
\(\hat {y}_i\)	Predicted target value
\(\loss (\by ,\hat {\by })\) or \(\mathcal {L}(y_i,\hat {y}_i)\)	Loss function
\(\lambda \)	Regularization parameter
\(\ba ^{[k]}\)	Activation of layer \(k\)
\(\bz ^{[k]}\)	Output of layer \(k\)
\(g_k(\cdot )\)	Activation function of layer \(k\)
\(\bth \) or \(\theta _i\)	Model parameters (general form)
\(\balpha \)	Kernel/dual coefficients vector
\(\be \)	Error/residual vector
\(\bepsilon \) or \(\epsilon _i\)	Noise vector/term
\(\bn \)	Noise vector (signal processing)
\(\bh \)	Impulse response / filter coefficients
\(\bP \)	Projection matrix
\(\bK \)	Kernel matrix
\(\bR \)	Autocorrelation matrix
\(\phi (\cdot )\)	Feature mapping / basis function
\(\alpha \)	Learning rate (gradient descent step size)

.
\(x\)	Sample set
\(\bar x\)	Sample mean
\(s_x^2\)	Sample variance (biased or unbiased)
\(s_x\)	Sample std (biased or unbiased)
\(s_{xy}\)	Sample covariance (biased or unbiased)
\(r_{xy}\)	Sample correlation coefficient
\(\mu \)	Population mean
\(\sigma ^2\)	Population variance
\(\sigma \)	Population standard deviation
\(\E [\cdot ]\)	Expectation operator
\(\Var [\cdot ]\)	Variance operator
\(\Cov [\cdot ]\)	Covariance operator

.
\(\omega \)	Angular frequency (discrete)
\(\theta \)	Phase angle
\(A\)	Amplitude
\(F\)	Frequency [Hz]
\(F_s\)	Sampling frequency
\(T\)	Period [sec]