main.tex

\documentclass[a0]{tumposter}

\usepackage[english]{babel}

\usepackage{blindtext}

\usepackage{multicol}
\usepackage{amsmath}
\usepackage{graphicx}

% for printing fontsizes
\usepackage{printlen}
\uselengthunit{mm}

\input{preamble.tex}

\usepackage[utf8]{inputenc}

\title{
	Early Classification for Agricultural Monitoring \\ from Satellite Time Series
	}
	
\author{
	Marc Rußwurm,\footnotemark[1] Romain Tavenard,\footnotemark[2] Sébastien Lefèvre,\footnotemark[2] Marco Körner\footnotemark[1]
	}

\header{
	Remote Sensing Technology \\
	TUM Department of Civil, Geo and Environmental Engineering \\
	Technical University of Munich
	}
	
\begin{document}
\maketitle

\renewcommand{\subsection}[1]{\textbf{#1}}

\begin{minipage}[t]{.65\textwidth}
	
	\section{Objective}
	
	
	\begin{tikzpicture}[scale=16.8]
	\node[label={[name=sat,text height=1.5ex,text depth=.25ex]Satellite Data}, anchor=north](x) at (-1,0){$\M{X} = (\V{x}_0, \V{x}_1, \dots , \V{x}_T)$};
	\node[below=of x, label={[yshift=1.3em, xshift=1.8em, font=\tiny, text=white]below:ESA Sentinel 2 Satellite}](s2){\includegraphics[width=11cm]{images/sentinel2}};
	
	
	\node[below=0em of s2, text width=11cm](eqbox){
		\small
		\begin{itemize}
			\item collected at regular temporal intervals of 2-3 days
			\item measurements of 13 spectral bands
			\item data available globalls
		\end{itemize} 
	};
	
	\node[label={[name=cm,text height=1.5ex,text depth=.25ex]{Early} {Classification} Model}, anchor=north](f) at (0,0){ $\yhat_t, {\deltat} = f(\xuptot)$};
	\node[below=0em of f, text width=13cm, font=\small](info){
		\begin{description}
		
		\item[$\xuptot$] observation until $t$  \\
		\item[$\yhat_t$] class prediction scores \\
		\item[$\deltat$] probability of stopping.
		
		\end{description}
		
	};
%	\node[below=of f](example){\input{images/example.tikz}};
	\node[circle, below=of info, text width=11cm, fill=tumbluedark, text=white](example){Classifying a \\ satellite time series \\ {\color{tumbluelight}\textbf{accurately}} \color{white} as {\color{ecolor}\textbf{early}} \color{white} as possible};
	
	
	\node[label={[name=ctm,text height=1.5ex,text depth=.25ex]Crop Type Labels}, anchor=north](y) at (1,0){
		$\V{y} = \small (y_\text{corn}, y_\text{barley}, \dots) \in \mathbb{R}^{13}$
	};
	\node[below=of y, label=below:crop type labels](labels){\includegraphics[width=11cm]{images/parcels}};
	\node[below=of labels, text width=11cm](lbls){
		\small
		\begin{itemize}\setlength\itemsep{.1em}
		\item European Common Agricultural Policy (CAP)
		\item collected yearly in entire Europe
%		\item slowly made publicly available
%		\item today, gathered on a national basis
%		\item in future harmonized within Europe's INSPIRE directive
		\end{itemize}
	};
	
	\draw[very thick, -{Stealth[scale=.5]}, tumblue] (x) -- (f);
	\draw[very thick, -{Stealth[scale=.5]}, tumblue] (f) -- (y);
	
	\coordinate(bottomleftcolumn) at (example.south -| s2);
	\coordinate(bottomrightcolumn) at (example.south -| lbls);
	
	\scoped[on background layer]
	{
		\node[fit=(ctm)(lbls)(bottomrightcolumn), fill=tumbluelight!20, inner sep=1em, rounded corners]{};
		\node[fit=(f)(example)(cm), fill=tumorange!20, inner sep=1em, rounded corners]{};
		\node[fit=(sat)(eqbox)(bottomleftcolumn), fill=tumbluelight!20, inner sep=1em, rounded corners]{};
	}	
	
	\end{tikzpicture}
	
	
	\section{Method}
	
	Based on previous work (Rußwurm et al., 2019) applied to crop type mapping from remote sensing data.
	
	%A network output to indicate a probability of stopping $\deltat$
	
	\begin{minipage}[t]{.49\textwidth}
		\subsection{Mechanism}
		
		\input{images/backprop_stopping_rule.tikz}
		\backpropstoppingrulefull
		
%	\begin{tikzpicture}
%		\node[fill=tumbluelight!20, rounded corners](a) at (0,0){
%						\input{images/backprop_stopping_rule.tikz}
%						\backpropstoppingrulefull
%					};
%		\node[right=of a]{\input{images/qualitative_example.tikz}};
%	\end{tikzpicture}
%	\vspace{-10em}

%	probability of not having classified before (known at training time)
%	$$P(t) = \deltat \cdot \prod_{\tau=0}^{t-1} 1 - p_\tau$$

	

	\end{minipage}
	\begin{minipage}[t]{.49\textwidth}
		
	
	\subsection{Loss function}
	
	composite loss function
	$$\mathcal{L}(\V{x}, \V{y}) = \sum_{t=0}^T P(t;\deltauptot) \mathcal{L}_t(\xuptot, \V{y})$$
	
	
	A Loss function including accuracy and earliness
	
	\begin{tikzpicture}[node distance=-.5em, draw=tumbluelight, rounded corners]
	
	%	\node(allloss){$\mathcal{L}(\V{x}, \V{y}) = \sum_{t=0}^T P(t;\deltauptot) \mathcal{L}_t(\xuptot, \V{y})$};
	%	
	\node(loss){$\mathcal{L}_t(\xuptot, \V{y})$};
	\node[right=of loss](equals){$=$};
	\node[right=of equals, fill=accuracycolor!20, rounded corners, label={[font=\small]Classification Loss}](classificationloss){$\alpha \mathcal{L}_c (\xuptot, \V{y})$};
	\node[right=of classificationloss](minus){$-$};
	\node[right=of minus, fill=earlinesscolor!20, rounded corners, label={[font=\small]Earliness Reward}](earlinessreward){$(1 - \alpha)\mathcal{R}_e(t, \ycorrect_t)$};
	
	\node[below=1em of classificationloss, text width=10cm, xshift=-4em](clossexp){$\mathcal{L}_c = -\log(\ycorrect_t)$ \\
		\vspace{.5em} 
		\tiny
		cross entropy loss for accurate classifications \par};
	
	\node[below=1em of earlinessreward, text width=13cm, xshift=-2em](erewardexp){$\mathcal{R}_e(t, \ycorrect_t) = \ycorrect_t \left(1 - \frac{t}{T}\right)$ \\
		\vspace{.5em}
		\tiny
		reduces loss for earlier classifications $1-\frac{t}{T}$ if the correct class $\ycorrect_t$ has been predicted \par};
	
	\draw[thick] (classificationloss) -- (clossexp);
	\draw[thick] (earlinessreward) -- (erewardexp);
	\end{tikzpicture}
	
	
	\end{minipage}

	{\tiny 
	Rußwurm, M., Lefèvre, S., Courty, N., Emonet, R., Körner, M., and Tavenard, R. End-to-end learning for early classification of time series. arXiv preprint arXiv:1901.10681, 2019.
	\par
	}

	\vspace{-1em}
	\section{Application}
	
	\begin{minipage}[t]{.5\textwidth}
	\subsection{Agriculture}
	
	\small
	
	\vspace{1em}
	
		\textbf{Early Crop Detection}
		\begin{itemize}
			\item early assessment of cultivated crops
			\item basis for early crop yield estimation
		\end{itemize}
		\vspace{.3em}
		
		\textbf{Extraction of Crop Phenology}
		\begin{itemize}
			\item extraction of vegetation specific events 
			\item monitoring time of classification
			\item regional or temporal variations
		\end{itemize}
		\vspace{.3em}
		
		\textbf{Generalization}
		\begin{itemize}
			\item end-to-end trainable
			\item no region-specific expert knowledge
			\item applicable globally
		\end{itemize}
	
	\end{minipage}
	\begin{minipage}[t]{.49\textwidth}
	\subsection{Dataset and Area of Interest} \par
	\begin{minipage}{.6\textwidth}
		\small
		\vspace{1em}
		Hollfeld region Bavaria
		\begin{itemize}
			\item 49k field parcels
			\item 6 main crop types
			\item covering 40km by 30 km
			\item central Germany
		\end{itemize}
		
		\vspace{1em}
	
		\tiny Challenge: Class imbalance \par
		\input{images/partition_histograms.tikz}
%		\vspace{-4em}\includegraphics[width=.3\textwidth]{images/holl.pdf}
	\end{minipage}
	\begin{minipage}{.35\textwidth}
%		\input{images/partition_histograms.tikz}
		\tiny {\color{tumblue}regions with labels} and \\ {\color{tumorange} location of dataset. \par} \par
		\includegraphics[width=.9\textwidth]{images/oberfrankenineurope.pdf}
		 
		 \tiny partition in {\color{traincolor} train}, {\color{validcolor} validation}, and {\color{evalcolor} evaluation} \par
		\includegraphics[width=.9\textwidth]{images/holl.pdf}
	\end{minipage}
	
	\end{minipage}

\end{minipage}
\hfill
\begin{minipage}[t]{.32\textwidth}
	\section{Results}
	
	\subsection{Qualitative Example} \par
	{\footnotesize Single example showing reflectance data $\M{x}$ and predictions $\yhat$ along with the stopping time $\tstop \sim \text{Ber}(\deltat)$. \par}
	
	\vspace{.5em}
	\input{images/example.tikz}
	\vspace{-.5em}
	
	\subsection{Losses during Training} \par
	{\footnotesize The combined loss $L_t$, as well as earliness $L_e$ and accuracy $L_e$ losses during training. \par}

	\vspace{.5em}	
	\input{images/loss-accuracyplots.tikz}
	\vspace{-1em}

	\subsection{Stopping Condition Parameterization} \par
	{\footnotesize Stopping times throughout the training grouped by crop category. The parameterization of early classification is learned for different crop types at different times during training. \par}
	
	\input{images/trainingstoppingclasses.tikz}
	
	\subsection{Balancing Earliness and Accuracy} \par
	{\footnotesize Evaluting the effect of the trade-off parameter $\alpha$ on the accuracy and earliness ($\tstop$). Runs repeated three times to evaluated the sability of the results. \par}
	
	\begin{table}
		
		\scriptsize
		\hspace{0em}\begin{tabular}{lcccccc}
			\toprule\small
			\textbf{$\alpha$} & accuracy & $\meantstop$  & precision & recall & $f_1$ & $\kappa$ \\
			\cmidrule(lr){0-0}\cmidrule(lr){1-1}\cmidrule(lr){2-2}\cmidrule(lr){3-3}\cmidrule(lr){4-4}\cmidrule(lr){5-5}\cmidrule(lr){6-6}\cmidrule(lr){7-7}
			.0 & .25 $\pm$ .22 & .10 $\pm$ .17 & .19 $\pm$ .20 & .25 $\pm$ .17 & .16 $\pm$ .20 & .12 $\pm$ .19 \\
			.2 & .81 $\pm$ .03 & .40 $\pm$ .02 & .70 $\pm$ .01 & .74 $\pm$ .01 & .71 $\pm$ .01 & .71 $\pm$ .04 \\
			.4 & .80 $\pm$ .09 & .47 $\pm$ .03 & .71 $\pm$ .02 & .74 $\pm$ .01 & .71 $\pm$ .02 & .71 $\pm$ .10 \\
			.6 & .85 $\pm$ .02 & .88 $\pm$ .07 & .73 $\pm$ .04 & .74 $\pm$ .03 & .73 $\pm$ .03 & .77 $\pm$ .03 \\
			.8 & .84 $\pm$ .01 & .93 $\pm$ .05 & .72 $\pm$ .02 & .75 $\pm$ .01 & .73 $\pm$ .02 & .76 $\pm$ .02 \\
			1.0 & .83 $\pm$ .03 & 1.00 $\pm$ .00 & .72 $\pm$ .03 & .75 $\pm$ .01 & .72 $\pm$ .03 & .75 $\pm$ .04 \\
			\bottomrule
		\end{tabular}
	
	
		%	\caption{Varying the weighting factor $\alpha$ for \emph{early reward} loss formulation (\cref{sec:earlynessreward}).}
		%	\label{tab:alpha}
		
	\end{table}
	
	
	\subsection{Extracting Vegetation Characteristics} \par
	
	{\footnotesize Stopping time per crop category reveals characteristic variations in type of vegetation confirmed by date of harvest (\druschdatum) from local authorities. \par} 
	\vspace{.5em}
	\input{images/classboxplots.tikz}
	
\end{minipage}

%\frame{
\begin{footer}
	\begin{minipage}{.15\textwidth}
		\textbf{ICML Workshop}\\
		\includegraphics[width=5cm]{images/AI4SG}
	\end{minipage}
	\begin{minipage}{.275\textwidth}
		\textbf{Technical University of Munich}\footnotemark[1]\\
		TUM Department of Civil, Geo and Env. \\
		Remote Sensing Technology \\
		Arcisstr. 21, 80333 Munich, Germany
	\end{minipage}
	\begin{minipage}{.275\textwidth}
		\textbf{IRISA-Obelix}\footnotemark[2]\\
		Université Bretagne Sud \\
		IRISA, UMR 6074 CNRS \\
		Campus de Tohannic, 56000 Vannes, France
		
	\end{minipage}
	\begin{minipage}{.2\textwidth}
		\textbf{Data \& Code} \\
		%		\vspace{1em}
		{github.com/rtavenar/early\_rnn} \\
		{twitter.com/MarcCoru} \\
		www.lmf.bgu.tum.de/vision
	\end{minipage}
	\begin{minipage}{.05\textwidth}
		\hfill\includegraphics[width=5cm]{images/qr-code}\\
		
	\end{minipage}

	
\end{footer}
%}

\end{document}