\documentclass[a4paper,10pt]{article}
\usepackage[utf8x]{inputenc}
\usepackage{xfrac}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{bigints}
\usepackage{mdframed}
\usepackage{setspace}
\usepackage{multirow}

\usepackage{tikz}


\newcommand{\mv}[1]{\ensuremath{{\boldsymbol #1}} }
\newcommand{\xv}{\mv{x} }
\newcommand{\Xv}{\mv{X} }
\newcommand{\Mm}{\mv{M} }
\newcommand{\Mmt}{\ensuremath{\tilde{\mv{M}}} }
\newcommand{\qv}{\mv{q} }
\newcommand{\Qv}{\mv{Q} }
\newcommand{\vv}{\mv{v} }
\newcommand{\Vv}{\mv{V} }
\newcommand{\cv}{\mv{c} }
\newcommand{\alphav}{\mv{\alpha} }
\newcommand{\betav}{\mv{\beta} }
\newcommand{\yv}{\mv{y} }
\newcommand{\Yv}{\mv{Y} }
\newcommand{\dv}{\mv{d} }
\newcommand{\Dv}{\mv{D} }
\newcommand{\uv}{\mv{u} }
\newcommand{\etav}{\mv{\eta} }
\newcommand{\etavt}{\ensuremath{\tilde{\mv{\eta}}} }
\newcommand{\Oset}{\ensuremath{\mathbf{O}} }
\newcommand{\Lset}{\ensuremath{\mathbf{L}} }
\newcommand{\Nset}{\ensuremath{\mathbf{N}} }

\newcommand{\sumprod}[1]{\ensuremath{\mathop{\sum\prod}_{#1}} }

\providecommand{\pa}[1]{\ensuremath{\mbox{pa}_{#1}}}
\providecommand{\ch}[1]{\ensuremath{\mbox{ch}_{#1}}}
\providecommand{\cp}[1]{\ensuremath{\mbox{si}_{#1}}}
\providecommand{\neigh}[1]{\ensuremath{\mbox{ne}_{#1}}}


\newcommand{\lambdav}{\mv{\lambda} }
\newcommand{\lambdavt}{\ensuremath{\tilde{\mv{\lambda}}} }
\newcommand{\nut}{\ensuremath{\tilde{\nu}} }

\newcommand{\muv}{\mv{\mu}}
\newcommand{\muvt}{\ensuremath{\tilde{\mv{\mu}}}}
\newcommand{\kv}{\mv{K}}
\newcommand{\kvt}{\ensuremath{\tilde{\mv{K}}}}

\newcommand{\munv}{\mv{\mu}_0}
\newcommand{\sigm}{\mv{\Sigma}}
\newcommand{\Pm}{\mv{P}}
\newcommand{\Bm}{\mv{B}}
\newcommand{\Vm}{\mv{V}}


\newcommand{\tr}{\ensuremath{\mbox{tr}}}
\newcommand{\vop}{\ensuremath{\mbox{vec}}}
\newcommand{\mop}{\ensuremath{\mbox{mat}}}

\definecolor{cabd5e6}{RGB}{171,213,230}

%opening
\title{A collection of identities for variational inference with exponential-family models}
\author{Dominik Endres, Kathrin Pabst, Anna-Lena Eckert, Raphael Schween \\ \texttt{dominik.endres@uni-marburg.de} }


\begin{document}

\maketitle

\begin{abstract}
This is a collection of identities useful for variational inference with exponential family distributions/densities.
All derviations were done by the authors, unless indicated otherwise. This does \textbf{not imply} that the results collected here
have not appeared in the literature before.
\textbf{DISCLAIMER}:  this collection is a work in progress. It is certainly incomplete and probably buggy. Bug-reports and contributions are most welcome, please email \texttt{dominik.endres@uni-marburg.de}.
\end{abstract}

\section{Exponential family distributions}

A distribution is said to belong to the exponential family, if it can be written in the form \cite{BishopMachLearn}:

\begin{equation}
 \label{expofam}
 p(\xv|\etav)=h(\xv)g(\etav)\exp(\etav^T \uv(\xv))
\end{equation}
where the random variates \xv may be discrete or continuous, the \textit{sufficient statistics} \uv are functions of the \xv, not necessarily 
of the same dimensionality. However, the \uv need to be linearly independent.
The \etav are the \textit{natural parameters}, one for each sufficient statistic.
The function $g(\etav)$ is the normalization constant (replace the integral with a sum for discrete \xv):

\begin{equation}
 \label{normeq}
 g(\etav) \int d\xv\; h(\xv) \exp(\etav^T \uv(\xv)) = 1
\end{equation}

\subsection{Moments}

Taking the gradient $\nabla$ w.r.t. to \etav on both sides of equation \ref{normeq}, we find:

\begin{equation}
 \label{nderv1}
  (\nabla g(\etav)) \underbrace{\int d\xv\; h(\xv) \exp(\etav^T \uv(\xv))}_{=\frac{1}{g(\etav)}} + \underbrace{g(\etav) \int d\xv\; h(\xv) \uv(\xv) \exp(\etav^T \uv(\xv))}_{\langle \uv(\xv) \rangle} = \mv{0}
\end{equation}
and thus the expectation $\langle \uv(\xv) \rangle $ is:
\begin{equation}
 \label{expectation_u}
 \boxed{\langle \uv(\xv) \rangle = - \frac{\nabla g(\etav)}{g(\etav)} = - \nabla \log(g(\etav))}
\end{equation}

Computing the derivative of the i-th component of the l.h.s. of eqn. \ref{nderv1} w.r.t. $\etav_j$ yields 
(noting that $\frac{\partial^2 \log(f(x,y))}{\partial x \partial y}=\sfrac{\frac{\partial^2 f(x,y)}{\partial x \partial y}}{f(x,y)}-\sfrac{\frac{\partial f(x,y)}{\partial x} \frac{\partial f(x,y)}{\partial y} }{f^2(x,y)}$):

\begin{eqnarray}
 \frac{\frac{\partial^2 g(\etav)}{\partial \eta_i \partial \eta_v}}{g(\etav)} - \frac{\frac{\partial g(\etav)}{\partial \eta_i} \frac{\partial g(\etav)}{\partial \eta_j} }{g^2(\etav)} +
 \frac{\partial g(\etav)}{\partial \eta_j} \frac{\langle u_i(\xv) \rangle}{g(\etav)} &=& - g(\etav) \int d\xv \; h(\xv) u_i(\xv) u_j(\xv) \exp(\etav^T \uv(\xv)) \nonumber \\
 \frac{\partial^2 \log(g(\etav))}{\partial \eta_i \partial \eta_j} - \langle u_i(\xv) \rangle \langle u_j(\xv) \rangle & = & -\langle u_i(\xv) u_j(\xv) \rangle \nonumber \\
 \Rightarrow \; \langle u_i(\xv) u_j(\xv) \rangle - \langle u_i(\xv) \rangle \langle u_j(\xv) \rangle &=& -\frac{\partial^2 \log(g(\etav))}{\partial \eta_i \partial \eta_j}
\end{eqnarray}
Denoting the Hessian by $\nabla\nabla$, the covariance matrix of $\uv(\xv)$ is thus given by
\begin{equation}
 \label{covariance}
 \boxed{\mbox{Cov}(\uv(\xv))=-\nabla\nabla \log(g(\etav))}
\end{equation}
Higher order moments can be computed via higher order derviatives.

\subsection{Maximum Likelihood} 
For maximum-likelihood approximations, we need the gradient of $\log(p(\xv|\etav))$ w.r.t. $\etav$:
\begin{equation}
 \nabla \log(p(\xv|\etav)) = \nabla \log(g(\etav)) + \uv(\xv) = -\langle \uv(\xv) \rangle + \uv(\xv).
\end{equation}
In other words, maximizing the likelihood (i.e. following the gradient towards higher likelihood values) amounts to making the expected value of the sufficient statistic more similiar to the actually observed
sufficient statistic. For second-order optimization methods, the Hessian matrix may be needed, which is given by eqn. \ref{covariance}. 
For maximum likelihood parameter estimates, assume we had observed $N$ i.i.d. datapoints $\xv^n$. The parameter estimate is obtained by solving
\begin{equation}
 \sum_{n=1}^N \nabla \log(p(\xv^n|\etav)) = 0 \Rightarrow \langle \uv(\xv) \rangle = \frac{\sum_{i=1}^N \uv(\xv^n)}{N}
\end{equation}
i.e. by setting the data mean of the sufficient statistics equal to the expectation.


\subsection{Entropy}

The (differential) entropy of \xv given \etav is defined as

\begin{equation}
 H(\xv|\etav) = -\int d\xv\; p(\xv|\etav) \log(p(\xv|\etav)).
\end{equation}
note that this is not the conditional entropy of \xv given \etav.
Using the definition of the exponential family distribution (eqn. \ref{expofam}) , this can be written as
\begin{equation}
 \label{entropy}
 H(\xv|\etav) = -\int d\xv\; p(\xv|\etav) \left(\log(g(\etav)) + \log(h(\xv)) + \etav^T \uv(\xv) \right) \nonumber
\end{equation}
and thus
\begin{equation}
  \boxed{H(\xv|\etav) = -\log(g(\etav)) - \langle \log(h(\xv)) \rangle - \etav^T \langle \uv(\xv) \rangle }
\end{equation}

\noindent
For the gradient of the entropy w.r.t. \etav we need
\begin{eqnarray}
 \frac{\partial}{\partial \eta_k} \; \etav^T \langle \uv(\xv) \rangle  &=& \langle u_k(\xv) \rangle + \sum_i \eta_i \frac{\partial \langle u_i(\xv) \rangle}{\partial \eta_k} \nonumber \\
 &=& \langle u_k(\xv) \rangle - \sum_i \eta_i \frac{\partial^2 \log(g(\etav))}{\partial \eta_i \partial \eta_k}.
\end{eqnarray}
Using eqn. \ref{covariance}, we find
\begin{equation}
 \nabla \etav^T \langle \uv(\xv) \rangle = \langle \uv(\xv) \rangle + \mbox{Cov}(\uv(\xv)) \cdot \etav
\end{equation}

\noindent
We also need
\begin{eqnarray}
 \frac{\partial \langle \log(h(\xv))\rangle}{\partial \eta_k} &=& \frac{\partial}{\partial \eta_k}\left( g(\etav) \int d\xv\; h(\xv) \log(h(\xv)) \exp(\eta^T \uv(\xv))\right) \nonumber \\
 &=& \frac{\partial g(\etav)}{\partial \eta_k} \frac{\langle \log(h(\xv))\rangle}{g(\etav)} + g(\etav) \int d\xv \; h(\xv) \log(h(\xv)) u_k(\xv) \exp(\etav^T \uv(\xv)) \nonumber \\
 &=& \frac{\partial \log(g(\etav))}{\partial \eta_k} \langle \log(h(\xv))\rangle + \langle \log(h(\xv)) u_k(\xv) \rangle \nonumber \\
 &=& -\langle u_k(\xv) \rangle \langle \log(h(\xv)) \rangle + \langle \log(h(\xv)) u_k(\xv) \rangle
\end{eqnarray}
and thus
\begin{equation}
 \nabla \langle \log(h(\xv))\rangle = - \langle \log(h(x)) \rangle \langle \uv(\xv) \rangle + \langle \log(h(\xv)) \uv(\xv) \rangle
\end{equation}


The gradient of the entropy w.r.t. the \etav is thus
\begin{equation}
 \nabla H(\xv|\etav) = \underbrace{-\nabla \log(g(\etav))}_{\langle \uv(\xv) \rangle}  - \nabla \langle \log(h(\xv)) \rangle - \nabla \; \etav^T \langle \uv(\xv) \rangle
\end{equation}

\begin{equation}
\label{egrad}
   \boxed{\nabla H(\xv|\etav) = \langle \log(h(\xv)) \rangle \langle \uv(\xv) \rangle - \langle \log(h(\xv)) \uv(\xv) \rangle - \mbox{Cov}(\uv(\xv)) \cdot \etav}
\end{equation}


\subsection{Kullback-Leibler divergence}

The KL-divergence of a distribution with parameters $\etavt$ to a distribution with parameters $\etav$ is:

\begin{eqnarray}
 D(p(\xv|\etavt)||p(\xv|\etav))&=&\int d\xv\; p(\xv|\etavt) \log\left(\frac{p(\xv|\etavt)}{p(\xv|\etav)} \right) \nonumber \\
 &=& \int d\xv\; p(\xv|\etavt) \left[ \log\left(p(\xv|\etavt)\right) - \log\left(p(\xv|\etav)\right) \right] \nonumber \\
 &=& - H(\xv|\etavt) - \int d\xv\; p(\xv|\etavt)  \log\left(p(\xv|\etav)\right)
\end{eqnarray}

For this, we compute the following expectation under $p(\xv|\etavt)$:
\begin{eqnarray}
 \int d\xv\; p(\xv|\etavt)  \log\left(p(\xv|\etav)\right) &=& \int d\xv\; p(\xv|\etavt) \left(\log(g(\etav)) + \log(h(\xv)) + \etav^T \uv(\xv) \right) \nonumber \\
 &=&  \log(g(\etav)) + \langle \log(h(\xv)) \rangle + \etav^T \langle \uv(\xv) \rangle
\end{eqnarray}

and thus, using eqn. \ref{entropy}:

\begin{equation}
 \boxed{D(p(\xv|\etavt)||p(\xv|\etav)) = \log(g(\etavt)) - \log(g(\etav)) + (\etavt^T-\etav^T) \langle \uv(\xv) \rangle}
\end{equation}

Its gradient w.r.t. \etavt is then (using eqn. \ref{expectation_u} and eqn. \ref{covariance})

\begin{eqnarray}
 \nabla D(p(\xv|\etavt)||p(\xv|\etav)) &=& \nabla \log(g(\etavt)) + \nabla \; (\etavt^T-\etav^T) \langle \uv(\xv) \rangle \nonumber \\
&=& -\langle \uv(\xv) \rangle + \langle \uv(\xv) \rangle + \nabla \langle \uv(\xv) \rangle (\etavt-\etav)
\end{eqnarray}

where $\nabla \langle \uv(\xv) \rangle$ acts on \uv component-wise, yielding a matrix $(ij)$ of derivatives $\frac{\partial u_i(\xv)}{\partial \etavt_j}$.
Thus

\begin{equation}
 \boxed{\nabla D(p(\xv|\etavt)||p(\xv|\etav)) = -\nabla\nabla \log(g(\etavt))  \cdot (\etavt-\etav) = \mbox{Cov}(\uv(\xv)) (\etavt-\etav)}
\end{equation}


\section{\label{conjugate}Conjugate priors on exponential family distributions}

For inference and learning in hierarchical models, conjugate priors on the parameters \etav are very useful, because inference/learning with i.i.d. observations 
translates into parameter updates (rather than complicated integrals).
The conjugate prior on an exponential family distribution (eqn. \ref{expofam}) is given by
\begin{equation}
 \label{cprior}
 p(\etav|\lambdav,\nu)=f(\lambdav,\nu) m(\etav) g(\etav)^\nu \exp(\nu \etav^T \lambdav)
\end{equation}
where the $\lambdav,\nu$ are the parameters of the p(oste)rior, $g(\etav)$ is the same function as above and $m(\etav)$ is an arbitrary positive function (different from $g(\etav)$). 
To see that this is a conjugate prior on $p(\xv|\etav)$, assume we had
observed $N$ datapoints $\xv_n$. The posterior of $\etav$ is then (using eqns. \ref{expofam} and \ref{cprior})
\begin{eqnarray}
 \label{postderv}
 p(\etav|\lambdav,\nu,\xv_{1:N}) &=& \frac{p(\xv_{1:N},\etav|\lambdav,\nu)}{p(\xv_{1:N}|\lambdav,\nu)} \nonumber \\
 &=& \frac{\prod_{n=1}^N p(\xv_n|\etav) \; p(\etav|\lambdav,\nu)}{\bigintssss d\etav \prod_{n=1}^N p(\xv_n|\etav) \; p(\etav|\lambdav,\nu)} \nonumber \\
 &=& \frac{\prod_{n=1}^N g(\etav)h(\xv_n) \exp(\etav^T \uv(\xv_n)) \cdot f(\lambdav,\nu) m(\etav) g(\etav)^\nu \exp(\nu \etav^T \lambdav) }{\bigintssss d\etav \prod_{n=1}^N g(\etav) h(\xv_n) \exp(\etav^T \uv(\xv_n)) \cdot f(\lambdav,\nu) m(\etav) g(\etav)^\nu \exp(\nu \etav^T \lambdav)} \nonumber \\
 &=& \frac{ \left[\prod_n h(\xv_n)\right] f(\lambdav,\nu) m(\etav) \; g(\etav)^{\nu+N} \exp\left(\etav^T \left(\nu\lambdav + \sum_n \uv(\xv_n)\right)\right) }{ \left[\prod_n h(\xv_n)\right] f(\lambdav,\nu) \bigintssss d\etav \; m(\etav) g(\etav)^{\nu+N} \exp\left(\etav^T \left(\nu\lambdav + \sum_n \uv(\xv_n)\right)\right)} \nonumber \\
 &=& \frac{ m(\etav) g(\etav)^{\nu+N} \exp\left(\etav^T \left(\nu\lambdav + \sum_n \uv(\xv_n)\right)\right) }{\bigintssss d\etav \; m(\etav) g(\etav)^{\nu+N} \exp\left(\etav^T \left(\nu\lambdav + \sum_n \uv(\xv_n)\right)\right)}
\end{eqnarray}

Note that this expression depends on the data $\xv_{1:N}$ only through $N$ and $\sum_n \uv(\xv_n)$. This is why the $\uv(\xv)$ are called \textit{sufficient statistics}: they contain all the information
about $\etav$ which we need from the data to determine the parameter posterior. A similar result holds for maximum-likelihood learning, see \cite{BishopMachLearn}. By introducting the \\ \vspace{0.2cm}

\begin{mdframed}[linewidth=1pt]
\begin{center}
\textit{\large posterior parameters}
\end{center}
\begin{eqnarray}
 \label{nuup}
 \tilde{\nu} &:=& \nu + N \\
 \label{lamup}
 \lambdavt &:=& \frac{\nu \lambdav + \sum_n \uv(\xv_n)}{\tilde{\nu}}
\end{eqnarray}
\end{mdframed}

\noindent
thus, $\nu \lambdav + \sum_N \uv(\xv_n)=\tilde{\nu}\lambdavt$. We furthermore identify $f(\lambdav,\nu)$ as the normalization constant of the p(oste)rior, i.e.

$$ f(\lambdavt,\tilde{\nu}) = \frac{1}{\bigintssss d\etav \; m(\etav)\, g(\etav)^{\tilde{\nu}} \exp\left(\etav^T \tilde{\nu}\lambdavt \right)}$$
and finally, plugging these identities back into eqn. \ref{postderv}, we obtain:
\begin{equation}
 \label{posterior}
 \boxed{p(\etav|\lambdav,\nu,\xv_{1:N})  = f(\lambdavt,\tilde{\nu})\, m(\etav)\, g(\etav)^{\tilde{\nu}} \exp\left( \etav^T \tilde{\nu}\lambdavt \right) = p(\etav|\lambdavt,\tilde{\nu}).}
\end{equation}

\noindent
In other words, given an exponential family observation model, i.i.d. data and a conjugate prior, we obtain posterior just by replacing the prior parameters according to eqns. \ref{nuup} and \ref{lamup}.
Furthermore, note that
\begin{itemize}
 \item According to eqn. \ref{nuup}, $\tilde{\nu}$ keeps track of the number of observed datapoints. Since it also contains prior information via $\nu$, it is referred to as a \textit{pseudocount}. 
 \item For large enough $N$, the posterior is unimodal, and the log-posterior is convex. The width of the maximum is monotonically decreasing in $\tilde{\nu}$ (Can be shown by computing the Hessian of the log-posterior). Hence, $\tilde{\nu}$ is also called the \textit{concentration parameter}.
 \item The posterior \lambdavt is just a weighted mean of the prior \lambdav and and the observed data.
 \item These posterior updates can be iterated, i.e. the accumulation of the sufficient statistics can be restarted at any point. The extreme case of updating after every datapoint, i.e. \textit{online learning},
 boils down to keeping track of this weighted average datapoint per datapoint.
\end{itemize}

\subsection{Maximum-a-posteriori (MAP) parameter estimates}

Instead of working with the full posterior of the natural parameters $\etav$ (eqn. \ref{posterior}), it is sometimes enough to use the parameter values which maximize the posterior, i.e. the numerator
of eqn. \ref{posterior} (the denominator does not depend on $\etav$ after the integration). Setting the derivative of the log of the numerator to zero, we find


\begin{eqnarray}
\hspace*{-0.5cm}
 \nabla_\etav \log(m(\etav)) + (\nu+N) \underbrace{\nabla_\etav \log(g(\etav))}_{-\langle \uv(\xv) \rangle, \mbox{eqn. \ref{expectation_u}}} + (\nu\lambdav + \sum_n \uv(\xv_n)) &\stackrel{!}{=}& 0 \nonumber \\
 \Rightarrow \frac{\nu \lambdav + \sum_n \uv(\xv_n)}{N+\nu} + \frac{\nabla_\etav \log(m(\etav))}{N+\nu}  &\stackrel{!}{=}& \langle \uv(\xv) \rangle
\end{eqnarray}
i.e. the posterior maximum is located at a point where the expected value of the natural parameters is equal to the quotient of the posterior parameters (eqns. \ref{nuup},\ref{lamup}) plus a term depending
on $m(\etav)$. The latter is often zero, since $m(\etav)=1$ for many distributions (see tables in section \ref{speccase}).


\subsection{Expectations}

Computing parameter expectations (i.e. of \etav and functions thereof) of a conjugate p(oste)rior can be done similar to the expectations of an exponential
family distribution. From the normalization equation

\begin{equation}
\label{priornorm}
 f(\lambdav,\nu)\int d\etav\; m(\etav)\, g(\etav)^\nu \exp\left( \nu \etav^T \lambdav \right) = 1
\end{equation}
follows 

\begin{eqnarray}
 \label{etaexp_deriv}
 \nabla_\lambdav \left[ f(\lambdav,\nu)\int d\etav\; m(\etav)\, g(\etav)^\nu \exp\left( \nu \etav^T \lambdav\right) \right] &=& 0 \nonumber \\
 \Rightarrow \nabla_\lambdav f(\lambdav,\nu) \underbrace{\int d\etav\; m(\etav)\, g(\etav)^\nu \exp\left( \nu \etav^T \lambdav\right)}_{f(\lambdav,\nu)^{-1}} &=&
 - f(\lambdav,\nu) \int d\etav\; m(\etav)\, g(\etav)^\nu \exp\left( \nu \etav^T \lambdav\right) \nu \etav \nonumber \\
 \Rightarrow \frac{\nabla_\lambdav f(\lambdav,\nu)}{f(\lambdav,\nu)} &=& - \nu \langle \etav \rangle
\end{eqnarray}
and thus
\begin{equation}
 \label{etaexp}
 \boxed{\langle \etav \rangle = - \frac{\nabla_\lambdav \log(f(\lambdav,\nu))}{\nu}}
\end{equation}
Likewise, from the derivative w.r.t. $\nu$ we find, noting that $\frac{\partial g(\etav)^\nu}{\partial \nu}=\log(g(\etav))g(\etav)^\nu$:

\begin{equation}
 \label{gexp}
 \boxed{\left\langle \log(g(\etav)) \right\rangle + \lambdav^T \langle \etav \rangle = - \frac{\partial \log(f(\lambdav,\nu))}{\partial \nu} }
\end{equation}

\noindent
For the second moments, take the derivatives of $\langle \etav \rangle$ (see eqn. \ref{etaexp_deriv}):
\begin{eqnarray}
 \frac{\partial \langle \etav \rangle}{\partial \lambda_j} &=& \frac{\partial}{\partial \lambda_j} \left[ f(\lambdav,\nu) \int d\etav\; m(\etav)\, g(\etav)^\nu \exp\left( \nu \etav^T \lambdav\right) \etav \right] \nonumber \\
&=& \frac{\partial f(\lambdav,\nu) }{\partial \lambda_j} \underbrace{\int d\etav\; m(\etav)\, g(\etav)^\nu \exp\left( \nu \etav^T \lambdav\right) \etav}_{=\frac{\langle \etav \rangle}{f(\lambdav,\nu)}} \nonumber \\
& & + f(\lambdav,\nu) \int d\etav\; m(\etav)\, g(\etav)^\nu \exp\left( \nu \etav^T \lambdav\right) \nu \etav \eta_j \nonumber \\
&=& \frac{\partial \log(f(\lambda,\nu))}{\partial \lambda_j} \langle \etav \rangle + \nu \langle \etav \eta_j \rangle \nonumber \\
&=& -\nu \langle \eta_j \rangle \langle \etav \rangle + \nu \langle \etav \eta_j \rangle \\
\Rightarrow \frac{1}{\nu} \frac{\partial \langle \eta_k \rangle}{\partial \lambda_j}  &=& \langle \eta_j \eta_k \rangle - \langle \eta_j \rangle \langle \eta_k \rangle
\end{eqnarray}
and thus

\begin{equation}
 \boxed{ \mbox{Cov}(\etav) = -\frac{\nabla_\lambdav \nabla_\lambdav \log(f(\lambdav,\nu))}{\nu^2}  }
\end{equation}


\noindent
Likewise, computing the derivative of eqn. \ref{gexp} on both sides yields
\begin{equation}
 \frac{\partial \langle \log(g(\etav)) \rangle}{\partial \nu} = - \frac{\partial^2 \log\left(f(\lambdav,\nu))\right)}{\partial \nu^2}- \lambdav^T \frac{\partial \langle \etav \rangle}{\partial \nu}
\end{equation}
where
\begin{eqnarray}
 \frac{\partial \langle \log(g(\etav)) \rangle}{\partial \nu} &=& \frac{\partial \log(f(\lambdav,\nu))}{\partial \nu} \langle \log(g(\etav)) \rangle + \left\langle \log(g(\etav))^2 \right\rangle + \lambdav^T \left\langle \log(g(\etav)) \etav \right\rangle \nonumber \\
 &=& \left\langle \log(g(\etav))^2 \right\rangle - \left\langle \log(g(\etav)) \right\rangle^2 +  \lambdav^T \left\langle \log(g(\etav)) \etav \right\rangle  - \lambdav^T \left\langle \log(g(\etav)) \right\rangle \langle \etav \rangle\nonumber \\
 &=& \mbox{Var}(\log(g(\etav))) + \mbox{Cov}(\log(g(\etav)),\lambdav^T \etav) \nonumber \\
 \label{eetaderv}
 \frac{\partial \langle \eta_i \rangle}{\partial \nu} &=& \frac{\partial \log(f(\lambda,\nu))}{\partial \nu}\langle \eta_i \rangle + \langle \eta_i \log(g(\etav)) \rangle + \lambdav^T \langle \etav \eta_i \rangle \\
 &=& \langle \eta_i \log(g(\etav)) \rangle - \langle \eta_i \rangle \langle \log(g(\etav)) \rangle + \lambdav^T \langle \etav \eta_i \rangle - \lambdav^T \langle \etav \rangle \langle \eta_i \rangle \nonumber \\
 &=& \mbox{Cov}(\log(g(\etav)),\eta_i) + \mbox{Cov}(\lambdav^T \etav, \eta_i) \nonumber \\
\Rightarrow \lambdav^T \frac{\partial \langle \etav \rangle}{\partial \nu} &=& \mbox{Cov}(\log(g(\etav)),\lambdav^T \etav) + \mbox{Var}(\lambdav^T \etav)
\end{eqnarray}
and thus, noting that $\mbox{Var}(x+y) = \mbox{Var}(x) + \mbox{Var}(y) + 2 \;\mbox{Cov}(x,y)$:
\begin{equation}
 \boxed{\mbox{Var}\left( \log(g(\etav)) + \lambdav^T \etav \right)= - \frac{\partial^2 \log\left(f(\lambdav,\nu))\right)}{\partial \nu^2} }
\end{equation}

\noindent
Another expectation that can be computed from the normalization constant $f(\lambdav,\nu)$ is
\begin{equation}
 \langle g(\etav)^k \rangle = f(\lambdav,\nu) \int d\etav\; m(\etav)\, g(\etav)^{\nu+k} \exp\left( \nu \etav^T \lambdav \right). \\ 
\end{equation}
To evaluate the integral, choose new parameters $\nu^\prime, \lambdav^\prime$ such that
\begin{eqnarray}
 \nu^\prime &=& \nu+k \\
 \nu^\prime \lambdav^\prime &=& \nu \lambdav \Rightarrow \lambdav^\prime=\frac{\nu}{\nu^\prime} \lambdav.
\end{eqnarray}
With these parameters, the integral is in exponential family normal form, and thus
\begin{equation}
 \langle g(\etav)^k \rangle = \frac{f(\lambdav,\nu)}{f(\lambdav^\prime,\nu^\prime)}
\end{equation}

\begin{equation}
 \boxed{\langle g(\etav) \rangle = \frac{f(\lambdav,\nu)}{f(\frac{\nu}{\nu+1}\lambdav,\nu+1)}}
\end{equation}

\begin{equation}
 \boxed{\mbox{Var}(g(\etav)) = \frac{f(\lambdav,\nu)}{f(\frac{\nu}{\nu+2}\lambdav,\nu+2)} - \left[\frac{f(\lambdav,\nu)}{f(\frac{\nu}{\nu+1}\lambdav,\nu+1)}\right]^2 }
\end{equation}

\subsection{Predictive distribution, entropy and log-likelihood}

Let $\alpha>0$. The predictive distribution and related quantities can be derived from the integral

\begin{eqnarray}
 \int d\etav\; p(\xv|\etav)^\alpha p(\etav | \lambdav,\nu) &=& h(\xv)^\alpha f(\lambdav,\nu) \int d\etav\, g(\etav)^\alpha \exp\left(\alpha \etav^T \uv(\xv) \right) m(\etav) g(\etav)^\nu \exp\left( \nu \etav^T \lambdav \right) \nonumber \\
 &=& h(\xv)^\alpha f(\lambdav,\nu) \int d\etav\, m(\etav) g(\etav)^{\nu+\alpha} \exp\left( \nu \etav^T \lambdav + \alpha \etav^T \uv(\xv) \right) \nonumber \\
 &=& h(\xv)^\alpha f(\lambdav,\nu) \int d\etav\, m(\etav) g(\etav)^{\nu+\alpha} \exp\left( (\nu+\alpha) \etav^T \frac{\nu\lambdav + \alpha \uv(\xv)}{\nu+\alpha} \right) \nonumber \\
 \langle p(\xv|\etav)^\alpha) \rangle_{p(\etav|\lambdav,\nu)}&=&  h(\xv)^\alpha f(\lambdav,\nu) \frac{1}{f\left(\frac{\nu\lambdav + \alpha \uv(\xv)}{\nu+\alpha},\nu+\alpha\right)}
\end{eqnarray}
where the last line follows from the normalization equation \ref{priornorm}. For $\alpha=1$, the integral on the l.h.s. is the expectation of $p(\xv|\etav)$ under the prior:

\begin{equation}
 \label{expected_p_under_prior}
 \boxed{p(\xv|\lambdav,\nu) = h(\xv) \frac{f(\lambdav,\nu)}{f\left(\frac{\nu\lambdav + \uv(\xv)}{\nu+1},\nu+1\right)}}
\end{equation}

\noindent
Differentiating with respect to $\alpha$ yields:

\begin{eqnarray}
 \frac{\partial}{\partial \alpha} \langle p(\xv|\etav)^\alpha) \rangle_{p(\etav|\lambdav,\nu)} &=& \langle \log(p(\xv|\etav)) p(\xv|\etav)^\alpha \rangle_{p(\etav|\lambdav,\nu)} \nonumber \\
 &=& \log(h(\xv))\; h(\xv)^\alpha \frac{f(\lambdav,\nu)}{f\left(\frac{\nu\lambdav + \alpha \uv(\xv)}{\nu+\alpha},\nu+\alpha\right)} \nonumber \\
 - \frac{h(x)^\alpha f(\lambdav,\nu)}{f\left(\frac{\nu\lambdav + \alpha \uv(\xv)}{\nu+\alpha},\nu+\alpha\right)^2} &\cdot& \left[\frac{\nu}{(\nu+\alpha)^2}\nabla_{\lambdav^\prime} f(\lambdav^\prime,\nu^\prime)  (\uv(\xv)-\lambdav) + \frac{\partial f(\lambdav^\prime,\nu^\prime)}{\partial \nu^\prime}\right] \nonumber
\end{eqnarray}
where the derivatives are evaluated at $\lambdav^\prime=\frac{\nu\lambdav + \alpha \uv(\xv)}{\nu+\alpha}$ and $\nu^\prime=\nu+\alpha$.
For $\alpha=0$, we obtain the expected log-likelihood, using eqn. \ref{etaexp}:

\begin{equation}
 \label{expected_log_p_under_prior}
 \boxed{ \langle \log(p(\xv|\etav)) \rangle_{p(\etav|\lambdav,\nu)} = \log(h(x))+ \langle \etav^T \rangle  (\uv(\xv)-\lambdav) - \frac{\partial \log(f(\lambdav,\nu))}{\partial \nu}}
\end{equation} 
For $\alpha=1$, we obtain expectations of the form $\langle \log(p(\xv|\etav)) p(\xv|\etav)) \rangle_{p(\etav|\lambdav,\nu)}$, which are the terms required for the computation of the expected entropy
of $\xv$:

\begin{eqnarray}
 \lambdav^\prime &=& \frac{\nu\lambdav + \uv(\xv)}{\nu+1} \\
 \nu^\prime &=& \nu+1 \\
 \langle \log(p(\xv|\etav)) p(\xv|\etav) \rangle_{p(\etav|\lambdav,\nu)} &=& p(\xv|\lambdav,\nu) \left[\log(h(\xv)) + \frac{\nu}{\nu^\prime}\langle \etav^T\rangle_{\lambdav^\prime,\nu^\prime}  (\uv(\xv)-\lambdav) \right. \nonumber \\
 && -\left. \frac{\partial \log(f(\lambdav^\prime,\nu^\prime))}{\partial \nu^\prime} \right]
\end{eqnarray}

\subsection{Expected sufficient statistics}

The expectations of the sufficient statistics $\uv$ for fixed natural parameters $\etav$ is given by eqn. \ref{expectation_u}:

\begin{equation}
 \langle \uv(\xv) \rangle_{p(\xv|\etav)} = - \nabla \log(g(\etav))
\end{equation}
If $\eta$ is drawn from a conjugate prior, the expectation of $\uv(\xv)$ under the prior is given by averaging the r.h.s. over $p(\etav|\lambdav,\nu)$,
\begin{equation}
 \label{expectation_u_prior}
 \langle \uv(\xv) \rangle_{p(\xv|\etav)p(\etav|\lambdav,\nu)} = - \langle \nabla \log(g(\etav)) \rangle_{p(\etav|\lambdav,\nu)}
\end{equation}
To compute this expectation,
one can use the Divergence Theorem from vector calculus (Ostrogradsky-Gauss). For a differentiable vector field $\mathbf{f}(\xv)$, this theorem states that

\begin{equation}
 \int_V d\xv\; \nabla \cdot \mathbf{f}(\xv) = \oint_S d\mathbf{s}\; \mathbf{f}(\xv)
\end{equation}
where $S$ is the surface enclosing the volume $V$. 
As a special case, consider the field $\mathbf{f}(\xv) = \mathbf{c} z(\xv)$, with $\vec{c}$ a constant vector and $z(\xv)$ a smoothly differentiable scalar function.
Then, using $\nabla \cdot \mathbf{c} z(\xv) = \sum_i \frac{\partial c_i z(\xv)}{\partial x_i} = \mathbf{c}^T \nabla z(\xv)$ we find
\begin{equation}
 \mathbf{c}^T \int_V d\xv\; \nabla z(\xv) = \mathbf{c}^T \oint_S d\mathbf{s}\; \mathbf{z}(\xv)
\end{equation}
and since this holds for any $\mathbf{c}$, it follows that
\begin{equation}
  \int_V d\xv\; \nabla \vec{z}(\xv) = \oint_S d\mathbf{s}\; \mathbf{z}(\xv)
\end{equation}

This identity can be used to compute the expectation on the r.h.s. of eqn. \ref{expectation_u_prior} by integrating the gradient of $p(\etav|\lambdav,\nu)$:
\begin{eqnarray}
 \int d\etav \nabla_\etav p(\etav|\lambdav,\nu) &=& f(\lambdav,\nu) \int d\etav \nabla_\etav \left(m(\etav)g(\etav)^\nu \exp\left(\nu\etav^T \lambdav \right) \right) \nonumber \\
 &=& f(\lambdav,\nu) \int d\etav\; \left(\frac{\nabla_\etav m(\etav)}{m(\etav)} + \nu \frac{\nabla_\etav g(\etav)}{g(\etav)} + \nu \lambda\right) p(\etav|\lambdav,\nu) \nonumber \\
 &=& \langle \nabla_\etav \log(m(\etav)) \rangle_{p(\etav|\lambdav,\nu)} + \nu \langle \nabla_\etav \log(g(\etav)) \rangle_{p(\etav|\lambdav,\nu)} + \nu\lambdav \nonumber \\
 &=& \oint_S d\mathbf{s}\; p(\etav|\lambdav,\nu)
\end{eqnarray}
where the surface $S$ encloses the range of $\etav$.
Hence, the expectation is:
\begin{equation}
 \label{expected_u_under_prior}
 \boxed{\langle \uv(\xv) \rangle_{p(\xv|\etav)p(\etav|\lambdav,\nu)} = \lambdav + \frac{ \langle \nabla_\etav \log(m(\etav)) \rangle_{p(\etav|\lambdav,\nu)}-\oint d\mathbf{s}\; p(\etav|\lambdav,\nu)}{\nu}}
\end{equation}
If $\etav \in \mathbb{R}^D$ with no further constraints, then $p(\etav|\lambdav,\nu) \to 0$ on the surface of the range of $\etav$, since $p(\etav|\lambdav,\nu) $ has to be normalizable.
Hence, the surface integral must be zero. This is e.g. the case for the Multinomial distribution and the Poisson distribution. 
Furthermore, if $m(\etav)=\mbox{const.}$, then the gradient vanishes (Dirichlet,Gamma, Stick-breaking for $c_i=0$, see tables in appendix).
In those cases, the above expression simplifies to

\begin{equation}
 \langle \uv(\xv) \rangle_{p(\xv|\etav)p(\etav|\lambdav,\nu)} = \lambdav
\end{equation}

The expectation for the multivariate Gaussian is also computable, see appendix.




\subsection{Maximum likelihood}

For maximum-likelihood approximations, we need the gradient of $\log(p(\etav|\lambdav,\nu))$ w.r.t. $\lambdav$ and $\nu$:
\begin{eqnarray}
 \nabla_{\lambdav} \log(p(\etav|\lambdav,\nu)) &=& \nabla_{\lambdav} \log(f(\lambdav,\nu)) + \nu\etav \nonumber \\
 &=& -\nu \langle \etav \rangle + \nu\etav = \nu (\etav - \langle \etav \rangle ) \\
 \frac{\partial}{\partial \nu} \log(p(\etav|\lambdav,\nu)) &=& \frac{\partial f(\lambdav,\nu)}{\partial \nu} + \log(g(\etav)) + \etav^T \lambdav \nonumber \\
 &=& (\log(g(\etav)) - \langle \log(g(\etav)) \rangle) + \lambdav^T (\etav - \langle \etav \rangle)
\end{eqnarray}
Similar to the gradient of the exponential family distributions, this gradient points towards the actual value of \etav and away from the expectation.




\subsection{Entropy}

The differential entropy (not the conditional entropy) of \etav given \lambdav and $\nu$ is

\begin{eqnarray}
 \label{priorent1}
 H(\etav|\lambdav,\nu) & = & - f(\lambdav,\nu) \int d\etav\; m(\etav)\, g(\etav)^\nu \exp(\nu \etav^T \lambdav) \left[ \log(f(\lambdav,\nu)) + \log(m(\etav)) + \nu \log(g(\etav)) + \nu \etav^T \lambdav \right] \nonumber \\
&=& -\log(f(\lambdav,\nu)) - \langle \log(m(\etav)) \rangle - \nu \left[ \langle \log(g(\etav)) \rangle + \lambdav^T \langle \etav \rangle \right] 
\end{eqnarray}
where the expectations are w.r.t. the p(oste)rior eqn. \ref{cprior}. Using eqn. \ref{gexp}, this can be rewritten as
\begin{equation}
 \label{priorent2}
 \boxed{H(\etav|\lambdav,\nu) = -\log(f(\lambdav,\nu)) -\langle \log(m(\etav)) \rangle + \nu \frac{\partial \log(f(\lambdav,\nu))}{\partial \nu}}
\end{equation}

The derivates of this entropy are therefore:

\begin{equation}
 \boxed{ \frac{\partial H(\etav|\lambdav,\nu)}{\partial \nu} = \nu \frac{\partial^2 \log \left(f(\lambdav,\nu)\right)}{\partial \nu^2} -\frac{\partial \langle \log(m(\etav)) \rangle}{\partial \nu}}
\end{equation}

and (using eqn. \ref{eetaderv} and \ref{etaexp}):

\begin{eqnarray}
 \nabla_\lambdav H(\etav|\lambdav,\nu)  &=& - \nabla_\lambdav \log(f(\lambdav,\nu)) + \nu \nabla_\lambdav \frac{\partial \log(f(\lambdav,\nu))}{\partial \nu} - \nabla_\lambdav \langle \log(m(\etav)) \rangle \nonumber \\
 &=& \nu \langle \etav \rangle + \nu \frac{\partial}{\partial \nu} \nabla_\lambda \log(f(\lambdav,\nu)) - \nabla_\lambdav \langle \log(m(\etav)) \rangle \nonumber \\
 &=& \nu \langle \etav \rangle - \nu \frac{\partial}{\partial \nu} \nu \langle \etav \rangle - \nabla_\lambdav \langle \log(m(\etav)) \rangle \nonumber \\
 &=& \nu \langle \etav \rangle - \nu \langle \etav \rangle \nonumber - \nu^2 \frac{\partial \langle \etav \rangle}{\partial \nu} - \nabla_\lambdav \langle \log(m(\etav)) \rangle \\
 &=& -\nu^2 \left[ \mbox{Cov}(\log(g(\eta)),\etav) + \mbox{Cov}(\lambda^T \etav,\etav) \right] - \nabla_\lambdav \langle \log(m(\etav)) \rangle \nonumber
\end{eqnarray}

\begin{equation}
 \boxed{\nabla_\lambdav H(\etav|\lambdav,\nu)  = - \nabla_\lambdav \log(f(\lambdav,\nu)) + \nu \nabla_\lambdav \frac{\partial \log(f(\lambdav,\nu))}{\partial \nu} - \nabla_\lambdav \langle \log(m(\etav)) \rangle}
\end{equation}

\begin{equation}
 \boxed{\nabla_\lambdav H(\etav|\lambdav,\nu)  =  -\nu^2 \frac{\partial \langle \etav \rangle}{\partial \nu} - \nabla_\lambdav \langle \log(m(\etav)) \rangle}
\end{equation}

\subsection{Kullback-Leibler divergence}

The KL-divergence of a distribution with parameters $\lambdavt,\tilde{\nu}$ to a distribution with parameters $\lambdav,\nu$ is given by

\begin{eqnarray}
 D(p(\etav|\lambdavt,\nut)||p(\etav|\lambdav,\nu)) &=& \int d\etav \; p(\etav|\lambdavt,\nut) \left[ \log(p(\etav|\lambdavt,\nut)) - \log(p(\etav|\lambdav,\nu)) \right] \nonumber \\
&=& -H(\etav|\lambdavt,\nut) - \int  d\etav \; p(\etav|\lambdavt,\nut)\log(p(\etav|\lambdav,\nu)) 
\end{eqnarray}
The second term on the r.h.s. is given by (expectations w.r.t $p(\etav|\lambdavt,\nut)$)
\begin{equation}
 \int  d\etav \; p(\etav|\lambdavt,\nut)\log(p(\etav|\lambdav,\nu))  = \log(f(\lambdav,\nu)) + \langle \log(m(\etav)) \rangle + \nu \langle \log(g(\etav)) \rangle + \nu \lambdav \langle \etav \rangle
\end{equation}
and thus, using eqn. \ref{priorent1} and \ref{gexp} we find
\begin{equation}
\label{KLprior}
\boxed{D(p(\etav|\lambdavt,\nut)||p(\etav|\lambdav,\nu)) = \log\left(\frac{f(\lambdavt,\nut)}{f(\lambdav,\nu)}\right) - (\nut-\nu) \frac{\partial \log(f(\lambdavt,\nut))}{\partial \nut} + (\lambdavt^T - \lambdav^T) \nu \langle \etav \rangle}
\end{equation}

\noindent
The derivatives are:
\begin{eqnarray}
 \nabla_{\lambdavt} D(p(\etav|\lambdavt,\nut)||p(\etav|\lambdav,\nu)) &=& \nabla_{\lambdavt} \log(f(\lambdavt,\nut)) - (\nut-\nu) \frac{\partial}{\partial \nut} \nabla_{\lambdavt} \log(f(\lambdavt,\nut)) \nonumber \\
 && + \nu \langle \etav \rangle - \frac{\nu}{\nut} (\lambdavt^T-\lambdav^T) \nabla_{\lambdavt} \nabla_{\lambdavt} \log(f(\lambdavt,\nut)) \nonumber \\
&=& - \nut \langle \etav \rangle + (\nut-\nu) \frac{\partial}{\partial\nut} \; \nut\langle\etav\rangle \nonumber \\
&& + \nu \langle \etav \rangle - \frac{\nu}{\nut} (\lambdavt^T-\lambdav^T) \nabla_{\lambdavt}^2 \log(f(\lambdavt,\nut)) \nonumber \\
&=& \nut (\nut - \nu) \frac{\partial \langle \etav \rangle}{\partial \nut} - \frac{\nu}{\nut} (\lambdavt^T-\lambdav^T) \nabla_{\lambdavt}^2 \log(f(\lambdavt,\nut)) \nonumber \\
&=& \nut (\nut - \nu) \frac{\partial \langle \etav \rangle}{\partial \nut} + \nu (\lambdavt^T-\lambdav^T) \nabla_{\lambdavt} \langle \etav \rangle
\end{eqnarray}

\begin{equation}
 \boxed{\nabla_{\lambdavt} D(p(\etav|\lambdavt,\nut)||p(\etav|\lambdav,\nu))=\nut (\nut - \nu) \frac{\partial \langle \etav \rangle}{\partial \nut} + \nu (\lambdavt^T-\lambdav^T) \nabla_{\lambdavt} \langle \etav \rangle}
\end{equation}
where $\nabla_{\lambdavt} \langle \etav \rangle$ is a matrix with entries $\left(\nabla_{\lambdavt} \langle \etav \rangle\right)_{i,j}=\frac{\partial \langle \etav \rangle_i}{\partial \tilde{\lambda}_j}$.
Likewise,

\begin{equation}
\boxed{\frac{\partial}{\partial \nut} D(p(\etav|\lambdavt,\nut)||p(\etav|\lambdav,\nu))= -(\nut-\nu) \frac{\partial^2 f(\lambdavt,\nut)}{\partial \nut^2}- \nu(\lambdavt^T-\lambdav^T)\frac{\partial \langle \etavt \rangle}{\partial \nut}}
\end{equation}





\section{Variational approximations with exponential family distributions}



\subsection{Variational inference with conjugate p(oste)riors}


In variational inference, we replace an intractable distribution (or density) $q(\Xv|\dv)$ (i.e. one where marginals and conditionals are hard to compute) with a tractable, factorized approximation $q(\Xv)$.
\dv is the observed data. Strictly speaking, $q(\Xv)=q(\Xv|\dv)$ but it is customary to omit writing this conditioning, since it is only approximate.
The approximation is linked to the correct distribution via the variational bound also 'evidence lower bound (ELBO)':
\begin{eqnarray}
 \log(q(\dv)) &=& \log\left( \sum_\xv q(\dv,\xv) \right) = \log\left( \sum_\xv q(\xv) p(\dv|\xv) \frac{p(\xv)}{p(\xv)} \right) \nonumber \\
 &\geq & \int_\xv q(\xv) \log\left(   p(\dv|\xv) \frac{p(\xv)}{q(\xv)} \right) = \left\langle \log p(\dv|\xv) \right\rangle_{q(\Xv)} - D(q(\Xv)||p(\Xv)) \nonumber \\
\Rightarrow L(q,\dv) &=& \left\langle \log p(\dv|\xv) \right\rangle_{q(\Xv)} - D(q(\Xv)||p(\Xv)) \leq \log(p(\dv)) \label{elbo}
\end{eqnarray}
where the second line follows from Jensen's inequality for convex functions and the definition of the Kullback-Leibler divergence. $\mathcal{L}(q,\dv)$ is a lower bound on the log-marginal-likelihood,
which we try to maximize w.r.t. $q(\xv)$. The resulting $q(\xv)$ is an approximate version of the correct posterior $p(\xv|\dv)$ which will be exact iff $p(\xv|\dv)$ is contained in the 
class of distributions which can be modeled by $q(\xv)$. In that case (and only in that case), the bound will be tight.

In the following, we will derive the posterior update rules for the case $p(\Xv)$ is conjugate to $q(\dv|\Xv)$ and both are in the exponential family. We also assume that $q(\Xv)$ is conjugate to the likelihood, such that posterior updated reduce to parameter updates, like in section \ref{conjugate}. Lastly, assume that the data are comprised of $N$ i.i.d. observations, i.e. $p(\dv|\Xv)=\prod_{i=1}^N p(\dv_i|\Xv)$.
We use a generalized version of the ELBO, which has an inverse temperature parameter $\beta \geq 0$:

\begin{equation}\label{elbo-beta}
 L(q,\dv) = \left\langle \log p(\dv|\xv) \right\rangle_{q(\Xv)} - \beta D(q(\Xv)||p(\Xv)) \leq \log(P(\dv))
\end{equation}

\noindent $\beta \neq 1$ can be used to model deviations from optimal inference, or for stochastic updating in minibatches etc..
Denote the prior parameters with $\nu,\lambdav$ and the posterior parameters with $\nut,\lambdavt$.

The expected log-likelihood under the posterior $\left\langle \log p(\dv|\xv) \right\rangle_{q(\Xv)}$ for $N$ datapoints can then be computed from eqn. (\ref{expected_log_p_under_prior}):


\begin{equation}\label{variational-log-likelihood}
 \langle \log(p(\dv|\etav)) \rangle_{q(\etav|\lambdavt,\nut)} = \sum_{i=1}^N \,\log(h(\dv_i))+ \langle \etav^T \rangle  ( \sum_{i=1}^N \uv(\dv_i)-N\,\lambdavt) - N\,\frac{\partial \log(f(\lambdavt,\nut))}{\partial \nut}
\end{equation} 

\noindent
where $\langle \etav^T \rangle$ is given by eqn. (\ref{etaexp}) and the KL-divergence $ D(q(\Xv)||p(\Xv)) $ by eqn. (\ref{KLprior}). To maximize eqn. (\ref{elbo-beta}) w.r.t. the posterior parameters $\lambdavt,\nut$, we will rewrite the elbo as difference between one part that does not depend on the posterior parameters, and a KL-divergence between the posterior, and a distribution in the same exponential family as the posterior that depends on parameters $\lambdav^\prime, \nu^\prime$ Because the KL-divergence is zero exactly if the two distributions that enter it are pointwise equal, we can then compute the maximal ELBO by setting $\lambdavt=\lambda^\prime$ and $\nu^\prime=\nut$. 

\begin{eqnarray}\label{elbo-beta-long}
  L(q,\dv) &=& \left\langle \log p(\dv|\xv) \right\rangle_{q(\Xv)} - \beta D(q(\Xv)||p(\Xv)) \nonumber \\
  &=&  \sum_{i=1}^N \,\log(h(\dv_i))+ \langle \etav^T \rangle  \left( \sum_{i=1}^N \uv(\dv_i)-N\,\lambdavt\right) - N\,\frac{\partial \log(f(\lambdavt,\nut))}{\partial \nut} \nonumber \\
  &-& \beta \left[\log\left(\frac{f(\lambdavt,\nut)}{f(\lambdav,\nu)}\right) - (\nut-\nu) \frac{\partial \log(f(\lambdavt,\nut))}{\partial \nut} + (\lambdavt^T - \lambdav^T) \nu \langle \etav \rangle \right] \nonumber \\
  &=& \sum_{i=1}^N \,\log(h(\dv_i)) - \beta\, \log\left(\frac{f(\lambdavt,\nut)}{f(\lambdav,\nu)}\right) \nonumber \\
  &+&  \langle \etav^T \rangle \left( \sum_{i=1}^N \uv(\dv_i) + \beta\nu\lambdav-(N+\beta\nu)\lambdavt \right) \nonumber \\
  &-& (N+\beta\nu - \beta\nut ) \frac{\partial \log(f(\lambdavt,\nut))}{\partial \nut} 
\end{eqnarray}

\noindent
Define
\begin{mdframed}[linewidth=1pt]
\begin{center}
 Posterior parameters for $\beta$-variational update
\end{center}
\begin{eqnarray} \label{beta-posterior-params}
 \nu^\prime &=& \frac{N}{\beta} + \nu \\
 \lambdav^\prime &=& \frac{\sum_{i=1}^N \uv(\dv_i) + \beta \nu \lambdavt}{N+\beta\nu} = \frac{\nu\lambdav + \sum_{i=1}^N \uv(\dv_i) / \beta}{\nu^\prime}
\end{eqnarray}
\end{mdframed}

\noindent and note the similarity of these definitions with the exact posterior updates eqn. (\ref{posterior}) -- all data-related quantities have been divided by $\beta$. Collecting terms in the ELBO eqn. (\ref{elbo-beta-long}) and plugging in these definitions, we find

\begin{eqnarray}
  L(q,\dv) &=& \sum_{i=1}^N \,\log(h(\dv_i)) - \beta\, \log\left(\frac{f(\lambdavt,\nut)}{f(\lambdav,\nu)}\right) \nonumber \\
  &+& \beta \nu^\prime \langle \etav^T \rangle \left( \lambdav^\prime - \lambdavt \right) \nonumber \\
  &-& \beta (\nut - \nu^\prime) \frac{\partial \log(f(\lambdavt,\nut))}{\partial \nut} 
\end{eqnarray}

\noindent Comparing the last two lines to the expression for the KL divergence eqn. (\ref{KLprior}), we find that up to a $\log\left(\frac{f(\lambdavt,\nut)}{f(\lambdav^\prime,\nu\prime)}\right)$ term and a factor $\beta$, these lines are a KL-divergence. Inserting and subtracting this term, we find

\begin{eqnarray}
  L(q,\dv) &=& \sum_{i=1}^N \,\log(h(\dv_i)) - \beta\, \log\left(\frac{f(\lambdavt,\nut)}{f(\lambdav,\nu)}\right) + \beta\, \log\left(\frac{f(\lambdavt,\nut)}{f(\lambdav^\prime,\nu^\prime)}\right)
  \nonumber \\
  &-& \beta D(q(\Xv|\lambdavt,\nut)||p(\Xv|\lambdav^\prime,\nu^\prime)) \nonumber \\
  &=& \sum_{i=1}^N \,\log(h(\dv_i)) - \beta\, \log\left(\frac{f(\lambdav^\prime,\nu^\prime)}{f(\lambda,\nu)}\right) \label{opt-beta-elbo} \\
  &-& \beta D(q(\Xv|\lambdavt,\nut)||p(\Xv|\lambdav^\prime,\nu^\prime))
\end{eqnarray}

\noindent which is maximal  if $D(q(\Xv|\lambdavt,\nut)||p(\Xv|\lambdav^\prime,\nu^\prime))=0$, which happens if and only if $\lambdavt=\lambdav^\prime$ and $\nu^\prime=\nut$. Thus, the maximal ELBO after the posterior update is given by eqn. (\ref{opt-beta-elbo}), which is achieved for the parameters given in eqns. (\ref{beta-posterior-params}).

Now we derive an expression for the expected log-likelihood eqn. (\ref{variational-log-likelihood}) that depends only on the p(oste)rior parameters. This is possible because these parameters are computed from the sufficient statistics (which are by definition sufficient to determine the likelihood). Rewrite the posterior parameters as $N=\beta(\nu^\prime - \nu)$ and $\beta ( \lambdav^\prime \nu^\prime - \lambdav \nu )$ and substitute these expressions into the log-likelihood, then

\begin{equation}
 \langle \log(p(\dv|\etav)) \rangle_{q(\etav|\lambdavt,\nut)} = \sum_{i=1}^N \,\log(h(\dv_i))+ \beta \nu  \langle \etav^T \rangle (\lambdav^\prime - \lambdav) - \beta (\nu^\prime - \nu)  \frac{\partial \log(f(\lambdav^\prime,\nu^\prime))}{\partial \nu^\prime} 
\end{equation}






\subsection{Variational message passing}

\begin{figure}

\small
\begin{tikzpicture}[y=0.80pt, x=0.8pt,yscale=-0.28, xscale=0.28, inner sep=0pt, outer sep=0pt]
\begin{scope}% layer1
  % path3004
  \path[shift={(78.57143,-25.71429)},draw=black,miter limit=4.00,line
    width=2.000pt] (165.7143,153.0765)arc(-0.018:180.018:42.142857 and
    39.286)arc(-180.018:0.018:42.142857 and 39.286) -- cycle;

  % text3774
  \path[fill=black] (178.57143,140.93361) node[above right] (text3774) {$X_1$};

  % path3004-5
  \path[shift={(436.42857,-24.71428)},draw=black,fill=cabd5e6,miter
    limit=4.00,line width=2.000pt] (165.7143,153.0765)arc(-0.018:180.018:42.142857
    and 39.286)arc(-180.018:0.018:42.142857 and 39.286) -- cycle;

  % text3774-3
  \path[fill=black] (536.42859,141.93361) node[above right] (text3774-3) {$X_2$};

  % path3004-1
  \path[shift={(72.14286,372.14287)},draw=black,fill=cabd5e6,miter limit=4.00,line
    width=2.000pt] (165.7143,153.0765)arc(-0.018:180.018:42.142857 and
    39.286)arc(-180.018:0.018:42.142857 and 39.286) -- cycle;

  % text3774-31
  \path[fill=black] (172.14285,538.79077) node[above right] (text3774-31) {$X_6$};

  % path3004-1-8
  \path[shift={(437.85715,377.85715)},draw=black,miter limit=4.00,line
    width=2.000pt] (165.7143,153.0765)arc(-0.018:180.018:42.142857 and
    39.286)arc(-180.018:0.018:42.142857 and 39.286) -- cycle;

  % text3774-31-5
  \path[fill=black] (537.85718,544.50507) node[above right] (text3774-31-5) {$X_7$};

  % path3004-1-0
  \path[shift={(255.0,180.7143)},draw=black,miter limit=4.00,line width=2.000pt]
    (165.7143,153.0765)arc(-0.018:180.018:42.142857 and
    39.286)arc(-180.018:0.018:42.142857 and 39.286) -- cycle;

  % text3774-31-2
  \path[fill=black] (355,347.36218) node[above right] (text3774-31-2) {$X_3$};

  % path3004-1-01
  \path[shift={(560.71429,170.71429)},draw=black,fill=cabd5e6,miter
    limit=4.00,line width=2.000pt] (165.7143,153.0765)arc(-0.018:180.018:42.142857
    and 39.286)arc(-180.018:0.018:42.142857 and 39.286) -- cycle;

  % text3774-31-8
  \path[fill=black] (660.71429,337.36218) node[above right] (text3774-31-8) {$X_5$};

  % path3004-1-6
  \path[shift={(-67.85714,170.7143)},draw=black,miter limit=4.00,line
    width=2.000pt] (165.7143,153.0765)arc(-0.018:180.018:42.142857 and
    39.286)arc(-180.018:0.018:42.142857 and 39.286) -- cycle;

  % text3774-31-86
  \path[fill=black] (32.142864,337.36218) node[above right] (text3774-31-86) {$X_4$};

  % path6558
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.600pt, arrows=->] (78.6325,356.7649) -- (172.7961,492.2452);

  % path6560
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.600pt, arrows=->] (178.0916,159.6260) -- (79.7655,291.5269);

  % path6562
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.600pt, arrows=->] (581.4947,162.1608) -- (662.7910,289.9921);

  % path6564
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.600pt, arrows=->] (663.8907,358.1778) -- (581.8236,496.5466);

  % path6566
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.600pt, arrows=->] (228.4045,158.0893) -- (352.3098,303.0636);

  % path6568
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.600pt, arrows=->] (533.2133,158.6924) -- (405.3582,303.4606);

  % path6570
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.600pt, arrows=->] (350.5453,363.1306) -- (223.7404,495.8795);

  % path6572
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.600pt, arrows=->] (406.1358,363.5086) -- (533.8642,501.2157);

\end{scope}
\end{tikzpicture}
\begin{tikzpicture}[y=0.80pt, x=0.8pt,yscale=-0.30, xscale=0.30, inner sep=0pt, outer sep=0pt]
\begin{scope}% layer1
  \begin{scope}[shift={(0,72.0)}]% g6794
    \begin{scope}[shift={(0,-28.0)}]% g6537
      % path3004
      \path[shift={(78.57143,-25.71429)},draw=black,miter limit=4.00,line
        width=2.000pt] (165.7143,153.0765) .. controls (165.7143,174.7734) and
        (146.8463,192.3622) .. (123.5714,192.3622) .. controls (100.2966,192.3622) and
        (81.4286,174.7734) .. (81.4286,153.0765) .. controls (81.4286,131.3796) and
        (100.2966,113.7908) .. (123.5714,113.7908) .. controls (146.8463,113.7908) and
        (165.7143,131.3796) .. (165.7143,153.0765) -- cycle;

    \end{scope}
    % text3774
    \path[fill=black] (178.57143,112.93361) node[above right] (text3774) {$X_1$};

  \end{scope}
  \begin{scope}[shift={(-16.0,72.99999)}]% g6788
    \begin{scope}[shift={(0,-30.0)}]% g6540
      % path3004-5
      \path[shift={(436.42857,-24.71428)},draw=black,fill=cabd5e6,miter
        limit=4.00,line width=2.000pt] (165.7143,153.0765) .. controls
        (165.7143,174.7734) and (146.8463,192.3622) .. (123.5714,192.3622) .. controls
        (100.2966,192.3622) and (81.4286,174.7734) .. (81.4286,153.0765) .. controls
        (81.4286,131.3796) and (100.2966,113.7908) .. (123.5714,113.7908) .. controls
        (146.8463,113.7908) and (165.7143,131.3796) .. (165.7143,153.0765) -- cycle;

    \end{scope}
    % text3774-3
    \path[fill=black] (536.42859,111.93361) node[above right] (text3774-3) {$X_2$};

  \end{scope}
  \begin{scope}[shift={(0,72.0)}]% g6818
    \begin{scope}[shift={(0,12.0)}]% g6552
      % path3004-1
      \path[shift={(72.14286,372.14287)},draw=black,fill=cabd5e6,miter limit=4.00,line
        width=2.000pt] (165.7143,153.0765) .. controls (165.7143,174.7734) and
        (146.8463,192.3622) .. (123.5714,192.3622) .. controls (100.2966,192.3622) and
        (81.4286,174.7734) .. (81.4286,153.0765) .. controls (81.4286,131.3796) and
        (100.2966,113.7908) .. (123.5714,113.7908) .. controls (146.8463,113.7908) and
        (165.7143,131.3796) .. (165.7143,153.0765) -- cycle;

    \end{scope}
    % text3774-31
    \path[fill=black] (172.14285,550.79077) node[above right] (text3774-31) {$X_6$};

  \end{scope}
  \begin{scope}[shift={(-2.0,76.28572)}]% g6824
    \begin{scope}[shift={(0,2.0)}]% g6546
      % path3004-1-8
      \path[shift={(437.85715,377.85715)},draw=black,miter limit=4.00,line
        width=2.000pt] (165.7143,153.0765) .. controls (165.7143,174.7734) and
        (146.8463,192.3622) .. (123.5714,192.3622) .. controls (100.2966,192.3622) and
        (81.4286,174.7734) .. (81.4286,153.0765) .. controls (81.4286,131.3796) and
        (100.2966,113.7908) .. (123.5714,113.7908) .. controls (146.8463,113.7908) and
        (165.7143,131.3796) .. (165.7143,153.0765) -- cycle;

    \end{scope}
    % text3774-31-5
    \path[fill=black] (537.85718,546.50507) node[above right] (text3774-31-5) {$X_7$};

  \end{scope}
  \begin{scope}[shift={(0.42855,64.0)},miter limit=4.00,line width=2.000pt]% g6549
    \begin{scope}% g7022
      % path3004-1-0
      \path[shift={(255.0,180.7143)},draw=black,miter limit=4.00,line width=2.000pt]
        (165.7143,153.0765) .. controls (165.7143,174.7734) and (146.8463,192.3622) ..
        (123.5714,192.3622) .. controls (100.2966,192.3622) and (81.4286,174.7734) ..
        (81.4286,153.0765) .. controls (81.4286,131.3796) and (100.2966,113.7908) ..
        (123.5714,113.7908) .. controls (146.8463,113.7908) and (165.7143,131.3796) ..
        (165.7143,153.0765) -- cycle;

    \end{scope}
  \end{scope}
  % text3774-31-2
  \path[fill=black] (355.42856,411.36218) node[above right] (text3774-31-2) {$X_3$};

  \begin{scope}[shift={(0,75.25001)}]% g6812
    \begin{scope}[shift={(10.0,0)}]% g6543
      % path3004-1-01
      \path[shift={(560.71429,170.71429)},draw=black,fill=cabd5e6,miter
        limit=4.00,line width=2.000pt] (165.7143,153.0765) .. controls
        (165.7143,174.7734) and (146.8463,192.3622) .. (123.5714,192.3622) .. controls
        (100.2966,192.3622) and (81.4286,174.7734) .. (81.4286,153.0765) .. controls
        (81.4286,131.3796) and (100.2966,113.7908) .. (123.5714,113.7908) .. controls
        (146.8463,113.7908) and (165.7143,131.3796) .. (165.7143,153.0765) -- cycle;

    \end{scope}
    % text3774-31-8
    \path[fill=black] (670.71429,337.36218) node[above right] (text3774-31-8) {$X_5$};

  \end{scope}
  \begin{scope}[shift={(0,75.25)}]% g6800
    \begin{scope}% g6555
      % path3004-1-6
      \path[shift={(-67.85714,170.7143)},draw=black,miter limit=4.00,line
        width=2.000pt] (165.7143,153.0765) .. controls (165.7143,174.7734) and
        (146.8463,192.3622) .. (123.5714,192.3622) .. controls (100.2966,192.3622) and
        (81.4286,174.7734) .. (81.4286,153.0765) .. controls (81.4286,131.3796) and
        (100.2966,113.7908) .. (123.5714,113.7908) .. controls (146.8463,113.7908) and
        (165.7143,131.3796) .. (165.7143,153.0765) -- cycle;

    \end{scope}
    % text3774-31-86
    \path[fill=black] (32.142864,337.36218) node[above right] (text3774-31-86) {$X_4$};

  \end{scope}
  \begin{scope}[shift={(0,78.72451)}]% g6763
    % rect6659
    \path[draw=black,miter limit=4.00,line width=2.407pt,rounded corners=0.0000cm]
      (22.2857,162.0765) rectangle (208.5637,224.9336);

    % text6661
    \path[fill=black] (36,214.93361) node[above right] (text6661) {$p(X_4|X_1)$};

  \end{scope}
  \begin{scope}[shift={(0,78.72449)}]% g6773
    % rect6659-9
    \path[draw=black,miter limit=4.00,line width=2.368pt,rounded corners=0.0000cm]
      (559.2415,162.0765) rectangle (739.5063,224.9336);

    % text6661-4
    \path[fill=black] (570.80359,214.93361) node[above right] (text6661-4)
      {$p(X_5|X_2)$};

  \end{scope}
  \begin{scope}[cm={{0.99475,0.0,0.0,1.0,(2.52416,78.0)}}]% g6768
    % rect6659-4
    \path[draw=black,miter limit=4.00,line width=2.902pt,rounded corners=0.0000cm]
      (242.2634,164.6479) rectangle (512.9190,227.5050);

    % text6711
    \path[fill=black] (263.42856,215.21933) node[above right] (text6711)
      {$p(X_3|X_1,X_2)$};

  \end{scope}
  \begin{scope}[cm={{0.94536,0.0,0.0,1.0,(-7.54417,89.14286)}}]% g6778
    % rect6659-4-3
    \path[draw=black,miter limit=4.00,line width=2.806pt,rounded corners=0.0000cm]
      (105.9366,380.9336) rectangle (359.1035,443.7907);

    % text6711-0
    \path[fill=black] (118.55968,432.50504) node[above right] (text6711-0)
      {$p(X_6|X_4,X_3)$};

  \end{scope}
  \begin{scope}[shift={(8.57142,84.85712)}]% g6783
    % rect6659-4-3-3
    \path[draw=black,miter limit=4.00,line width=2.783pt,rounded corners=0.0000cm]
      (432.5045,385.2193) rectangle (681.5181,448.0765);

    % text6711-0-6
    \path[fill=black] (446.59216,436.79074) node[above right] (text6711-0-6)
      {$p(X_7|X_5,X_3)$};

  \end{scope}
  \begin{scope}[shift={(96.37499,-131.14285)}]% g6861
    % rect6659-3
    \path[draw=black,miter limit=4.00,line width=1.927pt,rounded corners=0.0000cm]
      (56.2857,162.0765) rectangle (175.6195,224.9336);

    % text6661-9
    \path[fill=black] (64,211.93361) node[above right] (text6661-9) {$p(X_1)$};

  \end{scope}
  % path8003
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.600pt] (195.2325,532.9336) -- (195.4661,569.9343);

  % path8005
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.600pt] (559.8333,532.9336) -- (559.6370,569.9341);

  % path8007
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.600pt] (88.8013,423.3750) -- (152.3008,470.0765);

  % path8009
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.600pt] (342.9336,418.1239) -- (250.7812,470.0765);

  % path8011
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.600pt] (414.9076,418.3661) -- (505.1515,470.0764);

  % path8013
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.600pt] (660.9756,303.6581) -- (681.2365,361.6747);

  % path8015
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.600pt] (661.6734,423.9250) -- (601.1890,470.0765);

  % path8017
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.600pt] (93.6851,303.6581) -- (70.3804,362.1994);

  % path8019
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.600pt] (378.3543,305.5050) -- (378.7251,358.5059);

  % rect6659-3-6
  \path[draw=black,miter limit=4.00,line width=1.936pt,rounded corners=0.0000cm]
    (481.5179,30.9336) rectangle (602.0103,93.7908);

  % text6661-9-8
  \path[fill=black] (491.23212,80.790756) node[above right] (text6661-9-8)
    {$p(X_2)$};

  % path8048
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.600pt] (174.1825,200.7566) -- (136.0917,240.8010);

  % path8050
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.600pt] (573.4914,199.4256) -- (616.9722,240.8010);

  % path8052
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.600pt] (237.8673,192.2121) -- (324.2844,242.6479);

  % path8054
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.600pt] (508.8929,193.1027) -- (428.8860,242.6479);

  % path8058
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.600pt] (202.4479,93.7908) -- (202.2973,132.0767);

  % path8067
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.600pt] (542.4088,93.7908) -- (543.1943,132.0835);

\end{scope}

\end{tikzpicture}

\caption{\label{varexample}\textbf{Left:} Bayesian network with (undirected) loops. Exact sum-product can not be run on this graph. \textbf{Right}: corresponding
factor graph. For variational message passing, a node (e.g. $X_3$) needs to receive messages from all members of its Markov blanket, which are the variables connected to neighbouring factors  (for $X_3$: all other nodes).
through the connecting factors.}

\end{figure}





Denote the set of indexes of latent variables by \Lset, the set of indexes of observed variables with \Oset, and the set of all indexes by \Nset such that 
$\Lset \cap \Oset = \emptyset$ and  $\Lset \cup \Oset=\Nset$.
We consider a fully factorized approximation, i.e. one where the density of the latent variables 
$$Q(\xv)=\prod_{i \in \Lset} Q_i(x_i) = \prod_{i \in \Lset} Q(x_i)$$ 
is a product over distributions of individual variables. Strictly speaking, the notation in the middle is correct because there is
one density per variable. We omit the extra index and assume the reader knows what is meant.
Let the correct density be expressed as a Bayes net, $$P(\dv,\xv)=\prod_{j \in \Oset} P(d_j|\pa{X_j}) \prod_{i \in \Lset} P(x_i|\pa{X_i})$$ and $d_j$ is the observed data at node $X_j$.
Furthermore, to simplify notation, we introduce the "sum-product" symbol:
\begin{equation}
 \sumprod{i\in \mathbf{K}} Q(x_i) := \sum_{x_i: i \in \mathbf{K}} \prod_{i \in \mathbf{K}} Q(x_i)
\end{equation}

\noindent
The bound then is:

\begin{eqnarray}
 \label{varbound}
 L(Q,\dv) &=& \sum_\xv \prod_{i \in \Lset} Q(x_i) \left[ \sum_{j \in \Oset} \log(P(d_j|\pa{X_j}))+ \sum_{k \in \Lset} \log(P(x_k|\pa{X_k})) - \sum_{k \in \Lset} \log(Q(x_k)) \right] \nonumber \\
 &=& \sum_\xv \prod_{i \in \Lset} Q(x_i) \left[ \sum_{j \in \Oset} \log(P(d_j|\pa{X_j})) +\sum_{k \in \Lset} \log(P(x_k|\pa{X_k})) \right] \nonumber \\
 && - \sum_{i  \in \Lset} \sum_{x_i} Q(x_i) \log(Q(x_i)) \nonumber \\
 &=& \sum_{j\in \Oset} \;\; \sumprod{i\in \Lset \cap \pa{X_j}} Q(x_i) \log(p(d_j|\pa{X_j})) \nonumber \\
 & & + \sum_{k \in \Lset}\;\; \sumprod{i \in \Lset \cap \pa{X_k} \cup \{k\}} Q(x_i) \log(P(x_k|\pa{X_k})) \nonumber \\
 && - \sum_{i  \in \Lset} \sum_{x_i} Q(x_i) \log(Q(x_i))
\end{eqnarray}

\noindent
To find the $Q(\xv)$ that maximizes the bound, we take the derivative w.r.t. $q(\xv)$ and set it to zero. This is a necessary condition for a maximum, it can be shown that it is
sufficient, too. We furthermore impose the constraint that all $q(x_i)$ have to be distributions, i.e. $q(x_i)\geq 0$ and $\sum_{x_i} q(x_i) =1$. It will turn out that we do not have to impose
the first constraint, we do however need to make sure the second one is fulfilled. This can be achieved by a Lagrange multiplier for each distribution. The Lagrangian functional therefore is

\begin{equation}
 \mathcal{L}(Q,\dv) = L(Q,\dv) + \sum_{i \in \Lset} \tau_i \left( \sum_{x_i} \, Q(x_i) - 1 \right) 
\end{equation}


\noindent
A necessary condition for an extemum of $L(Q,\dv)$ is a stationary point of $\mathcal{L}(Q,\dv)$, i.e. the derivatives w.r.t. the 
components of $Q(\xv)$ have to vanish:


\begin{eqnarray}
 \label{varderiv}
 \frac{\delta \mathcal{L}(Q,\dv)}{\delta Q(X_m)} &=& \sum_{j \in \Oset \cap \ch{X_m}} \sumprod{i \in \Lset \cap \pa{X_j} \setminus \{m\}} Q(x_i) \log(P(d_j|\pa{X_j})) \nonumber \\
 & & + \sumprod{i\in \Lset \cap \pa{X_m}} Q(x_i) \log(P(x_m|\pa{X_m})) \nonumber \\
 & & + \sum_{k\in \Lset \cap \ch{X_m}} \sumprod{i\in \Lset \cap \pa{X_k} \cup \{k\} \setminus \{m\}} Q(x_i) \log(P(x_k|\pa{X_k})) \nonumber \\
 & & - \log(Q(x_m)) -1 + \tau_m
\end{eqnarray}
which we set to $0$ and solve for $Q(x_m)$ to find the optimal approximate posterior marginals.
To do so, we interpret the terms in eqn. \ref{varderiv} as \textit{messages} sent to node $X_m$ on the factor graph corresponding
to the Bayesian network (see fig. \ref{varexample} for an example).

Define the messages sent a variable node $X_m$ to a neighbouring factor node $f(X_m,\ldots)$ as just the variational posterior marginals
for unobserved nodes, and a 1-or-0 message for observed nodes:
\begin{eqnarray}
 \label{varmsg}
 \forall m\in \Lset : \mu_{X_m \rightarrow f(\ldots,X_m,\ldots)}(X_m) &=& Q(X_m) \\
 \forall j\in \Oset : \mu_{X_j \rightarrow f(\ldots,X_j,\ldots)}(X_j) &=& \delta_{x_j,d_j}
\end{eqnarray}
and the messages sent from a factor $f(\ldots,X_m,\ldots)$ depending on variables with indices $j\in \mathbf{F}, m \in \mathbf{F}$ to a neighbouring variable node as:
\begin{eqnarray}
 \label{facmsg}
 \mu_{f(\ldots,X_m,\ldots) \rightarrow X_m}(X_m) &=& \sumprod{j \in \mathbf{F} \setminus \{m\}} Q(x_j) f(\ldots,x_m,\ldots) \\
 &=& \sumprod{j \in \mathbf{F} \setminus \{m\}} f(\ldots,x_m,\ldots) \mu_{X_j \rightarrow f(\ldots,X_j,\ldots)}(x_j)
\end{eqnarray}
i.e. the average over the factor with respect to the posterior of all variables that connect to it, except for the variable where the message
is being sent to. With these message definitions,  eqn. \ref{varderiv} becomes

\begin{eqnarray}
 \frac{\delta L(Q,\dv)}{\delta Q(X_m)} &=& \sum_{j \in \Oset \cap \ch{X_m}} \mu_{\log(P(X_j|\pa{X_j})) \rightarrow X_m}(X_m) \nonumber \\
 & & + \mu_{\log(P(X_m|\pa{X_m}))\rightarrow X_m}(X_m) \nonumber \\
 & & + \sum_{k\in \Lset \cap \ch{X_m}} \mu_{\log(P(X_k|\pa{X_k}))\rightarrow X_m}(X_m) \nonumber \\
 & & - \log(Q(x_m)) -1 + \tau_m \\
 &=& \sum_{j \in \ch{X_m}} \mu_{\log(P(X_j|\pa{X_j})) \rightarrow X_m}(X_m) \nonumber \\
 & & + \mu_{\log(P(X_m|\pa{X_m}))\rightarrow X_m}(X_m) \nonumber \\
 & & - \log(Q(X_m)) -1 + \tau_m
\end{eqnarray}

\noindent
Defining $\exp(\tau_m-1)=\frac{1}{Z_m}$, we can solve for $Q(X_m)$ now:
\begin{equation}
 Q(X_m) = \frac{1}{Z_m} \exp\left( \sum_{j \in \ch{X_m}} \mu_{\log(P(X_j|\pa{X_j})) \rightarrow X_m}(X_m) + \mu_{\log(P(X_m|\pa{X_m}))\rightarrow X_m}(X_m) \right) 
\end{equation}
In other words, the variational posterior at variable node $X_m$ is computed by adding up all incoming messages, exponentiating, 
and normalizing (since $Z_i$ is computed from the Lagrange multiplier which enforces normalization). This message-passing scheme
has to be iterated until convergence, which is guaranteed since the bound $L(Q,D)$ is a Lyapunov function of the iteration dynamics.

Another way of deriving this algorithm without computing derivatives is via the KL divergence $D(Q(X)||\tilde{Q}(X))$. Recall that the KL divergence is
positive, and zero if and only if the distributions are equal everywhere. Assume again we wanted to maximize eqn. \ref{varbound} w.r.t. $Q(X_k)$.
To carry out this maximization, we only need to consider terms which depend on $Q(X_k)$, which in turn depend on the members of $X_k$'s Markov blanket:

\begin{eqnarray}
 \operatorname*{argmax}_{Q(X_k)}\left[L(Q,\dv)\right] &=& \operatorname*{argmax}_{Q(X_k)} \left[ \sum_{j \in \Oset \cap \ch{X_k}} \sumprod{i \in \Lset \cap \pa{X_j}} Q(x_i) \log(P(d_j|\pa{X_j})) \right. \nonumber \\
  & & + \sum_{j \in \Lset \cap \ch{X_k} \cup \{k\}} \sumprod{i \in \Lset \cap \pa{X_j} \cup \{j\}} \log(P(x_j|\pa{X_j})) \nonumber \\
  & & - \left. \sum_{x_k} Q(x_k) \log(Q(x_k)) \right]
\end{eqnarray}
Note that all terms on the r.h.s. include a factor $Q(x_k)$ and a summation over $x_k$. We can therefore pull it out:
\begin{eqnarray}
 \operatorname*{argmax}_{Q(X_k)}\left[L(Q,\dv)\right] &=& \operatorname*{argmax}_{Q(X_k)} \left[ \sum_{x_k} Q(x_k) \left( \sum_{j \in \Oset \cap \ch{X_k}} \sumprod{i \in \Lset \cap \pa{X_j} \setminus \{k\}} Q(x_i) \log(P(d_j|\pa{X_j})) \right. \right.\nonumber \\
  & & + \sumprod{i \in \Lset \cap \pa{X_k}} \log(P(x_k|\pa{X_k})) \nonumber \\
  & & \left. \left. + \sum_{j \in \Lset \cap \ch{X_k}} \sumprod{i \in \Lset \cap \pa{X_j} \cup \{j\} \setminus \{k\}} \log(P(x_j|\pa{X_j})) - \log(Q(x_k)) \right) \right]
\end{eqnarray}
With the message definitions (eqns. \ref{varmsg} -- \ref{facmsg}) the r.h.s. can be written as
\begin{eqnarray}
 \operatorname*{argmax}_{Q(X_k)}\left[L(Q,\dv)\right] &=& \operatorname*{argmax}_{Q(X_k)} \left[ \sum_{x_k} Q(x_k) \left( \sum_{j \in \Oset \cap \ch{X_k}} \mu_{\log(P(d_j|\pa{X_j})) \rightarrow X_k}(x_k) \right. \right.\nonumber \\ 
 & & + \mu_{\log(P(X_k|\pa{X_k})) \rightarrow X_k}(x_k) \nonumber \\
 & & + \left. \left. \sum_{j \in \Lset \cap \ch{X_k}} \mu_{\log(P(X_j|\pa{X_j}) \rightarrow X_k}(x_k) - \log(Q(x_k)) \right) \right] \nonumber \\
\end{eqnarray}i.e. we need the incoming messages from all neighbouring factor nodes to compute this expression. Note that the unions of the index sets of the sums
in the first and the last line are simply the indexes of all children of $X_k$, whereas the message on the second line is the incoming message
from the parents. Thus, define
\begin{equation}
 \log(U(x_k)) = \sum_{j \in \ch{X_k}} \mu_{\log(P(d_j|\pa{X_j})) \rightarrow X_k}(x_k) + \mu_{\log(P(X_k|\pa{X_k} \rightarrow X_k}(x_k)
\end{equation}
and let 
\begin{equation}
 U(x_k)=Z_k \tilde{Q}(x_k)
\end{equation}
with $\sum_{x_k} \tilde{Q}(x_k)=1$ and $Z_k>0$, i.e. $\tilde{Q}(X_k)$ is a probability distribution. With these definition we obtain
\begin{eqnarray}
 \operatorname*{argmax}_{Q(X_k)}\left[L(Q,\dv)\right] &=& \operatorname*{argmax}_{Q(X_k)} \left[ \sum_{x_k} Q(x_k) \log(U(x_k))-\log(Q(x_k)) \right] \nonumber \\
 &=& \operatorname*{argmax}_{Q(X_k)} \left[ \sum_{x_k} Q(x_k) \log(Z_k) + \sum_{x_k} Q(x_k) \log\left(\frac{\tilde{Q}(x_k)}{Q(x_k)}\right) \right] \nonumber \\
 &=& \log(Z_k) - \operatorname*{argmax}_{Q(X_k)} \left[ D(Q(X_k)||\tilde{Q}(X_k)) \right]
\end{eqnarray}
Since the KL-divergence is $\geq 0$, it follows that the variational bound $L(Q,\dv)$ is maximized if $Q(X_k)=\tilde{Q}(X_k)$. In other words,
to compute the optimal distribution at a given variable node given the distributions of the variables in its Markov blanket, do the following:
\begin{itemize}
 \item sum all incoming messages from neighbouring factor nodes,
 \item exponentiate,
 \item normalize.
\end{itemize}
The factor nodes collect messages from their neighbouring variable nodes, and compute messages by summing their log-factor over all variables except the one
where the message is being sent to, similar to sum-product message passing.

\subsection{Learning parameters with exponential family distributions}

\begin{figure}
 
\begin{tikzpicture}[y=0.80pt,x=0.80pt,yscale=-0.45, xscale=0.45, inner sep=0pt, outer sep=0pt]
\begin{scope}[shift={(0,-308.2676)}]% layer1
  \begin{scope}[shift={(0,527.95273)},fill=black]% flowRoot3755
  \end{scope}
  % path2985
  \path[draw=black,miter limit=4.00,line width=1.640pt] (76.4286,428.0050) ..
    controls (76.4286,442.4039) and (64.9159,454.0764) .. (50.7143,454.0764) ..
    controls (36.5127,454.0764) and (25.0000,442.4039) .. (25.0000,428.0050) ..
    controls (25.0000,413.6062) and (36.5127,401.9336) .. (50.7143,401.9336) ..
    controls (64.9159,401.9336) and (76.4286,413.6062) .. (76.4286,428.0050) --
    cycle;

  \begin{scope}[shift={(0,306.2676)}]% flowRoot5516
  \end{scope}
  \begin{scope}[shift={(8.0,308.2676)},fill=black]% flowRoot5615
    % path5644
    \path[fill] (273.1484,37.5183) -- (273.1484,41.9011) -- (275.1328,41.9011) ..
      controls (275.8672,41.9011) and (276.4349,41.7110) .. (276.8359,41.3308) ..
      controls (277.2370,40.9506) and (277.4375,40.4089) .. (277.4375,39.7058) ..
      controls (277.4375,39.0079) and (277.2370,38.4688) .. (276.8359,38.0886) ..
      controls (276.4349,37.7084) and (275.8672,37.5183) .. (275.1328,37.5183) --
      (273.1484,37.5183)(271.5703,36.2214) -- (275.1328,36.2214) .. controls
      (276.4401,36.2214) and (277.4271,36.5183) .. (278.0938,37.1120) .. controls
      (278.7656,37.7006) and (279.1015,38.5652) .. (279.1016,39.7058) .. controls
      (279.1016,40.8568) and (278.7656,41.7266) .. (278.0938,42.3152) .. controls
      (277.4271,42.9037) and (276.4401,43.1980) .. (275.1328,43.1980) --
      (273.1484,43.1980) -- (273.1484,47.8855) -- (271.5703,47.8855) --
      (271.5703,36.2214);

    % path5646
    \path[fill] (284.6172,35.7449) .. controls (283.9193,36.9428) and
      (283.4010,38.1277) .. (283.0625,39.2995) .. controls (282.7240,40.4714) and
      (282.5547,41.6589) .. (282.5547,42.8620) .. controls (282.5547,44.0652) and
      (282.7240,45.2579) .. (283.0625,46.4402) .. controls (283.4062,47.6173) and
      (283.9245,48.8022) .. (284.6172,49.9949) -- (283.3672,49.9949) .. controls
      (282.5859,48.7709) and (282.0000,47.5678) .. (281.6094,46.3855) .. controls
      (281.2240,45.2032) and (281.0312,44.0287) .. (281.0312,42.8620) .. controls
      (281.0312,41.7006) and (281.2240,40.5313) .. (281.6094,39.3542) .. controls
      (281.9948,38.1772) and (282.5807,36.9740) .. (283.3672,35.7449) --
      (284.6172,35.7449);

    % path5648
    \path[fill] (288.8047,46.5730) -- (288.8047,51.2136) -- (287.3594,51.2136) --
      (287.3594,39.1355) -- (288.8047,39.1355) -- (288.8047,40.4636) .. controls
      (289.1068,39.9428) and (289.4870,39.5574) .. (289.9453,39.3074) .. controls
      (290.4089,39.0522) and (290.9609,38.9246) .. (291.6016,38.9245) .. controls
      (292.6641,38.9246) and (293.5260,39.3464) .. (294.1875,40.1902) .. controls
      (294.8542,41.0339) and (295.1875,42.1433) .. (295.1875,43.5183) .. controls
      (295.1875,44.8933) and (294.8542,46.0027) .. (294.1875,46.8464) .. controls
      (293.5260,47.6902) and (292.6641,48.1120) .. (291.6016,48.1120) .. controls
      (290.9609,48.1120) and (290.4089,47.9870) .. (289.9453,47.7370) .. controls
      (289.4870,47.4818) and (289.1068,47.0938) ..
      (288.8047,46.5730)(293.6953,43.5183) .. controls (293.6953,42.4610) and
      (293.4766,41.6329) .. (293.0391,41.0339) .. controls (292.6068,40.4298) and
      (292.0104,40.1277) .. (291.2500,40.1277) .. controls (290.4896,40.1277) and
      (289.8906,40.4298) .. (289.4531,41.0339) .. controls (289.0208,41.6329) and
      (288.8047,42.4610) .. (288.8047,43.5183) .. controls (288.8047,44.5756) and
      (289.0208,45.4063) .. (289.4531,46.0105) .. controls (289.8906,46.6094) and
      (290.4896,46.9089) .. (291.2500,46.9089) .. controls (292.0104,46.9089) and
      (292.6068,46.6094) .. (293.0391,46.0105) .. controls (293.4766,45.4063) and
      (293.6953,44.5756) .. (293.6953,43.5183);

    % path5650
    \path[fill] (301.5469,43.4870) .. controls (300.3854,43.4871) and
      (299.5807,43.6199) .. (299.1328,43.8855) .. controls (298.6849,44.1511) and
      (298.4609,44.6042) .. (298.4609,45.2449) .. controls (298.4609,45.7553) and
      (298.6276,46.1615) .. (298.9609,46.4636) .. controls (299.2995,46.7605) and
      (299.7578,46.9089) .. (300.3359,46.9089) .. controls (301.1328,46.9089) and
      (301.7708,46.6277) .. (302.2500,46.0652) .. controls (302.7344,45.4975) and
      (302.9766,44.7449) .. (302.9766,43.8074) -- (302.9766,43.4870) --
      (301.5469,43.4870)(304.4141,42.8933) -- (304.4141,47.8855) --
      (302.9766,47.8855) -- (302.9766,46.5574) .. controls (302.6484,47.0886) and
      (302.2396,47.4818) .. (301.7500,47.7370) .. controls (301.2604,47.9870) and
      (300.6615,48.1120) .. (299.9531,48.1120) .. controls (299.0573,48.1120) and
      (298.3438,47.8620) .. (297.8125,47.3620) .. controls (297.2865,46.8568) and
      (297.0234,46.1824) .. (297.0234,45.3386) .. controls (297.0234,44.3542) and
      (297.3516,43.6121) .. (298.0078,43.1120) .. controls (298.6693,42.6121) and
      (299.6536,42.3621) .. (300.9609,42.3620) -- (302.9766,42.3620) --
      (302.9766,42.2214) .. controls (302.9766,41.5600) and (302.7578,41.0496) ..
      (302.3203,40.6902) .. controls (301.8880,40.3256) and (301.2786,40.1433) ..
      (300.4922,40.1433) .. controls (299.9922,40.1433) and (299.5052,40.2032) ..
      (299.0312,40.3230) .. controls (298.5573,40.4428) and (298.1016,40.6225) ..
      (297.6641,40.8620) -- (297.6641,39.5339) .. controls (298.1901,39.3308) and
      (298.7005,39.1798) .. (299.1953,39.0808) .. controls (299.6901,38.9766) and
      (300.1719,38.9246) .. (300.6406,38.9245) .. controls (301.9062,38.9246) and
      (302.8516,39.2527) .. (303.4766,39.9089) .. controls (304.1016,40.5652) and
      (304.4141,41.5600) .. (304.4141,42.8933);

    % path5652
    \path[fill] (306.5301,43.5037) -- (307.6321,43.5037) -- (309.5161,46.3221) --
      (311.4102,43.5037) -- (312.5122,43.5037) -- (310.0747,47.1448) --
      (312.6747,51.0855) -- (311.5727,51.0855) -- (309.4399,47.8608) --
      (307.2918,51.0855) -- (306.1848,51.0855) -- (308.8914,47.0382) --
      (306.5301,43.5037);

    % path5654
    \path[fill] (316.3642,35.6589) -- (316.3642,51.6589) -- (315.0360,51.6589) --
      (315.0360,35.6589) -- (316.3642,35.6589);

    % path5656
    \path[fill] (320.1220,45.9011) -- (321.7704,45.9011) -- (321.7704,47.8855) --
      (320.1220,47.8855) -- (320.1220,45.9011);

    % path5658
    \path[fill] (325.2157,45.9011) -- (326.8642,45.9011) -- (326.8642,47.8855) --
      (325.2157,47.8855) -- (325.2157,45.9011);

    % path5660
    \path[fill] (330.3095,45.9011) -- (331.9579,45.9011) -- (331.9579,47.8855) --
      (330.3095,47.8855) -- (330.3095,45.9011);

    % path5662
    \path[fill] (334.9735,35.7449) -- (336.2235,35.7449) .. controls
      (337.0048,36.9740) and (337.5881,38.1772) .. (337.9735,39.3542) .. controls
      (338.3642,40.5313) and (338.5595,41.7006) .. (338.5595,42.8620) .. controls
      (338.5595,44.0287) and (338.3642,45.2032) .. (337.9735,46.3855) .. controls
      (337.5881,47.5678) and (337.0048,48.7709) .. (336.2235,49.9949) --
      (334.9735,49.9949) .. controls (335.6662,48.8022) and (336.1819,47.6173) ..
      (336.5204,46.4402) .. controls (336.8642,45.2579) and (337.0360,44.0652) ..
      (337.0360,42.8620) .. controls (337.0360,41.6589) and (336.8642,40.4714) ..
      (336.5204,39.2995) .. controls (336.1819,38.1277) and (335.6662,36.9428) ..
      (334.9735,35.7449);

  \end{scope}
  \begin{scope}[shift={(0.71429,-197.21429)}]% g3771-6
    \begin{scope}[shift={(-10.0,190.0)}]% g3811
      % path2985-0
      \path[shift={(0,527.95273)},draw=black,miter limit=4.00,line width=1.640pt]
        (87.1429,64.0523) .. controls (87.1429,78.4511) and (75.6302,90.1237) ..
        (61.4286,90.1237) .. controls (47.2270,90.1237) and (35.7143,78.4511) ..
        (35.7143,64.0523) .. controls (35.7143,49.6534) and (47.2270,37.9809) ..
        (61.4286,37.9809) .. controls (75.6302,37.9809) and (87.1429,49.6534) ..
        (87.1429,64.0523) -- cycle;

      \begin{scope}[shift={(6.42857,531.52416)},fill=black]% flowRoot3763-8
        % path5666
        \path[fill] (49.5703,53.6902) -- (51.2656,53.6902) -- (54.1641,58.0261) --
          (57.0781,53.6902) -- (58.7734,53.6902) -- (55.0234,59.2917) --
          (59.0234,65.3542) -- (57.3281,65.3542) -- (54.0469,60.3933) --
          (50.7422,65.3542) -- (49.0391,65.3542) -- (53.2031,59.1277) --
          (49.5703,53.6902);

      \end{scope}
    \end{scope}
  \end{scope}
  % path2985-0-7
  \path[shift={(87.85714,430.02415)},draw=black,miter limit=4.00,line
    width=1.640pt] (87.1429,64.0523) .. controls (87.1429,78.4511) and
    (75.6302,90.1237) .. (61.4286,90.1237) .. controls (47.2270,90.1237) and
    (35.7143,78.4511) .. (35.7143,64.0523) .. controls (35.7143,49.6534) and
    (47.2270,37.9809) .. (61.4286,37.9809) .. controls (75.6302,37.9809) and
    (87.1429,49.6534) .. (87.1429,64.0523) -- cycle;

  \begin{scope}[shift={(92.0,427.59558)},fill=black]% flowRoot3763-8-1
    % path5441
    \path[fill] (57.6345,73.0683) .. controls (59.6025,73.0683) and
      (61.3785,72.4443) .. (62.7705,71.2923) .. controls (65.0985,69.3723) and
      (65.7225,66.6603) .. (65.7225,64.5963) .. controls (65.7225,62.6283) and
      (65.1225,59.8923) .. (62.7705,57.9963) .. controls (61.3785,56.8683) and
      (59.5545,56.2203) .. (57.6345,56.2203) .. controls (55.7145,56.2203) and
      (53.8905,56.8683) .. (52.4985,57.9963) .. controls (50.1465,59.8923) and
      (49.5465,62.6283) .. (49.5465,64.5963) .. controls (49.5465,66.6603) and
      (50.1705,69.3723) .. (52.4985,71.2923) .. controls (53.8905,72.4443) and
      (55.6665,73.0683) .. (57.6345,73.0683)(53.6025,59.3643) .. controls
      (54.4665,57.9243) and (56.0505,57.1563) .. (57.6345,57.1563) .. controls
      (59.2185,57.1563) and (60.8025,57.9243) .. (61.6665,59.3643) .. controls
      (62.1945,60.1803) and (62.3865,60.8043) .. (62.6265,61.8123) .. controls
      (63.1065,63.9723) and (63.0105,66.1803) .. (62.6265,67.7643) .. controls
      (62.5065,68.2683) and (62.1465,69.5403) .. (61.3785,70.4523) .. controls
      (60.3705,71.5563) and (58.9785,72.1323) .. (57.5865,72.1323) .. controls
      (56.2905,72.1323) and (54.8985,71.5563) .. (53.8905,70.4523) .. controls
      (53.1225,69.5403) and (52.7625,68.2683) .. (52.6425,67.7643) .. controls
      (52.2585,66.1803) and (52.1625,63.9723) .. (52.6425,61.8123) .. controls
      (52.8825,60.8043) and (53.0745,60.1803) .. (53.6025,59.3643)(56.1225,63.7563)
      .. controls (55.2345,63.7563) and (54.7785,63.2763) .. (54.5145,62.4843) --
      (54.1545,62.5083) .. controls (54.2505,63.2763) and (54.2985,63.9483) ..
      (54.2985,64.5483) .. controls (54.2985,65.1483) and (54.2505,65.8203) ..
      (54.1545,66.6123) -- (54.5145,66.6363) .. controls (54.7785,65.8203) and
      (55.2345,65.3643) .. (56.1225,65.3643) -- (59.1465,65.3643) .. controls
      (60.0345,65.3643) and (60.4905,65.8203) .. (60.7545,66.6363) --
      (61.1145,66.6123) .. controls (61.0185,65.8203) and (60.9705,65.1483) ..
      (60.9705,64.5483) .. controls (60.9705,63.9483) and (61.0185,63.2763) ..
      (61.1145,62.5083) -- (60.7545,62.4843) .. controls (60.4905,63.2763) and
      (60.0345,63.7563) .. (59.1465,63.7563) -- (56.1225,63.7563);

  \end{scope}
  % path3854
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.400pt, arrows=->] (50.9518,454.0754) -- (51.9053,558.7204);

  % path4300
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.400pt,arrows=->] (130.3714,511.7391) -- (71.0572,567.1281);

  % path2985-6
  \path[shift={(251.78571,362.16701)},draw=black,miter limit=4.00,line
    width=1.640pt] (87.1429,64.0523) .. controls (87.1429,78.4511) and
    (75.6302,90.1237) .. (61.4286,90.1237) .. controls (47.2270,90.1237) and
    (35.7143,78.4511) .. (35.7143,64.0523) .. controls (35.7143,49.6534) and
    (47.2270,37.9809) .. (61.4286,37.9809) .. controls (75.6302,37.9809) and
    (87.1429,49.6534) .. (87.1429,64.0523) -- cycle;

  \begin{scope}[shift={(261.78571,-202.21429)}]% g3771-6-1
    \begin{scope}[shift={(-10.0,190.0)}]% g3811-24
      % path2985-0-6
      \path[shift={(0,527.95273)},draw=black,miter limit=4.00,line width=1.640pt]
        (87.1429,64.0523) .. controls (87.1429,78.4511) and (75.6302,90.1237) ..
        (61.4286,90.1237) .. controls (47.2270,90.1237) and (35.7143,78.4511) ..
        (35.7143,64.0523) .. controls (35.7143,49.6534) and (47.2270,37.9809) ..
        (61.4286,37.9809) .. controls (75.6302,37.9809) and (87.1429,49.6534) ..
        (87.1429,64.0523) -- cycle;

      \begin{scope}[shift={(6.42857,531.52416)},fill=black]% flowRoot3763-8-8
        % path5672
        \path[fill] (49.5703,53.6902) -- (51.2656,53.6902) -- (54.1641,58.0261) --
          (57.0781,53.6902) -- (58.7734,53.6902) -- (55.0234,59.2917) --
          (59.0234,65.3542) -- (57.3281,65.3542) -- (54.0469,60.3933) --
          (50.7422,65.3542) -- (49.0391,65.3542) -- (53.2031,59.1277) --
          (49.5703,53.6902);

      \end{scope}
    \end{scope}
  \end{scope}
  % path2985-6-9
  \path[shift={(377.50002,439.16701)},draw=black,miter limit=4.00,line
    width=1.640pt] (87.1429,64.0523) .. controls (87.1429,78.4511) and
    (75.6302,90.1237) .. (61.4286,90.1237) .. controls (47.2270,90.1237) and
    (35.7143,78.4511) .. (35.7143,64.0523) .. controls (35.7143,49.6534) and
    (47.2270,37.9809) .. (61.4286,37.9809) .. controls (75.6302,37.9809) and
    (87.1429,49.6534) .. (87.1429,64.0523) -- cycle;

  % rect4677
  \path[draw=black,miter limit=4.00,line width=1.400pt] (262.4286,482.6479) --
    (366.0000,482.6479) -- (366.0000,525.5050) -- (262.4286,525.5050) -- cycle;

  \begin{scope}[fill=black]% text4679
    % path5677
    \path[fill] (273.1484,497.2807) -- (273.1484,501.6635) -- (275.1328,501.6635) ..
      controls (275.8672,501.6635) and (276.4349,501.4734) .. (276.8359,501.0932) ..
      controls (277.2370,500.7130) and (277.4375,500.1713) .. (277.4375,499.4682) ..
      controls (277.4375,498.7703) and (277.2370,498.2312) .. (276.8359,497.8510) ..
      controls (276.4349,497.4708) and (275.8672,497.2807) .. (275.1328,497.2807) --
      (273.1484,497.2807)(271.5703,495.9838) -- (275.1328,495.9838) .. controls
      (276.4401,495.9838) and (277.4271,496.2807) .. (278.0938,496.8745) .. controls
      (278.7656,497.4630) and (279.1015,498.3276) .. (279.1016,499.4682) .. controls
      (279.1016,500.6193) and (278.7656,501.4890) .. (278.0938,502.0776) .. controls
      (277.4271,502.6661) and (276.4401,502.9604) .. (275.1328,502.9604) --
      (273.1484,502.9604) -- (273.1484,507.6479) -- (271.5703,507.6479) --
      (271.5703,495.9838);

    % path5679
    \path[fill] (284.6172,495.5073) .. controls (283.9193,496.7052) and
      (283.4010,497.8901) .. (283.0625,499.0619) .. controls (282.7240,500.2338) and
      (282.5547,501.4213) .. (282.5547,502.6244) .. controls (282.5547,503.8276) and
      (282.7240,505.0203) .. (283.0625,506.2026) .. controls (283.4062,507.3797) and
      (283.9245,508.5645) .. (284.6172,509.7573) -- (283.3672,509.7573) .. controls
      (282.5859,508.5333) and (282.0000,507.3302) .. (281.6094,506.1479) .. controls
      (281.2240,504.9656) and (281.0312,503.7911) .. (281.0312,502.6244) .. controls
      (281.0312,501.4630) and (281.2240,500.2937) .. (281.6094,499.1166) .. controls
      (281.9948,497.9396) and (282.5807,496.7364) .. (283.3672,495.5073) --
      (284.6172,495.5073);

    % path5681
    \path[fill] (286.9141,495.9838) -- (288.6094,495.9838) -- (291.5078,500.3198) --
      (294.4219,495.9838) -- (296.1172,495.9838) -- (292.3672,501.5854) --
      (296.3672,507.6479) -- (294.6719,507.6479) -- (291.3906,502.6870) --
      (288.0859,507.6479) -- (286.3828,507.6479) -- (290.5469,501.4213) --
      (286.9141,495.9838);

    % path5683
    \path[fill] (300.2344,495.4213) -- (300.2344,511.4213) -- (298.9062,511.4213) --
      (298.9062,495.4213) -- (300.2344,495.4213);

    % path5685
    \path[fill] (305.8203,500.7729) -- (311.3359,500.7729) -- (311.3359,502.1010) --
      (305.8203,502.1010) -- (305.8203,500.7729)(308.5859,497.0541) .. controls
      (307.4401,497.0541) and (306.5312,497.4812) .. (305.8594,498.3354) .. controls
      (305.1823,499.1896) and (304.8438,500.3536) .. (304.8438,501.8276) .. controls
      (304.8438,503.2963) and (305.1797,504.4578) .. (305.8516,505.3120) .. controls
      (306.5286,506.1661) and (307.4401,506.5932) .. (308.5859,506.5932) .. controls
      (309.7318,506.5932) and (310.6406,506.1661) .. (311.3125,505.3120) .. controls
      (311.9792,504.4578) and (312.3125,503.2963) .. (312.3125,501.8276) .. controls
      (312.3125,500.3536) and (311.9792,499.1896) .. (311.3125,498.3354) .. controls
      (310.6406,497.4812) and (309.7318,497.0541) ..
      (308.5859,497.0541)(308.5859,495.7729) .. controls (310.2213,495.7729) and
      (311.5286,496.3224) .. (312.5078,497.4213) .. controls (313.4870,498.5151) and
      (313.9766,499.9838) .. (313.9766,501.8276) .. controls (313.9766,503.6661) and
      (313.4870,505.1323) .. (312.5078,506.2260) .. controls (311.5286,507.3250) and
      (310.2214,507.8745) .. (308.5859,507.8745) .. controls (306.9453,507.8745) and
      (305.6354,507.3276) .. (304.6562,506.2338) .. controls (303.6719,505.1401) and
      (303.1797,503.6713) .. (303.1797,501.8276) .. controls (303.1797,499.9838) and
      (303.6719,498.5125) .. (304.6562,497.4135) .. controls (305.6354,496.3198) and
      (306.9453,495.7729) .. (308.5859,495.7729);

    % path5687
    \path[fill] (316.7812,505.6635) -- (318.4297,505.6635) -- (318.4297,507.0073) --
      (317.1484,509.5073) -- (316.1406,509.5073) -- (316.7812,507.0073) --
      (316.7812,505.6635);

    % path5689
    \path[fill] (322.8984,506.3354) -- (322.8984,510.9760) -- (321.4531,510.9760) --
      (321.4531,498.8979) -- (322.8984,498.8979) -- (322.8984,500.2260) .. controls
      (323.2005,499.7052) and (323.5807,499.3198) .. (324.0391,499.0698) .. controls
      (324.5026,498.8146) and (325.0547,498.6870) .. (325.6953,498.6870) .. controls
      (326.7578,498.6870) and (327.6198,499.1088) .. (328.2812,499.9526) .. controls
      (328.9479,500.7963) and (329.2812,501.9057) .. (329.2812,503.2807) .. controls
      (329.2812,504.6557) and (328.9479,505.7651) .. (328.2812,506.6088) .. controls
      (327.6198,507.4526) and (326.7578,507.8745) .. (325.6953,507.8745) .. controls
      (325.0547,507.8745) and (324.5026,507.7495) .. (324.0391,507.4995) .. controls
      (323.5807,507.2442) and (323.2005,506.8562) ..
      (322.8984,506.3354)(327.7891,503.2807) .. controls (327.7891,502.2234) and
      (327.5703,501.3953) .. (327.1328,500.7963) .. controls (326.7005,500.1922) and
      (326.1042,499.8901) .. (325.3438,499.8901) .. controls (324.5833,499.8901) and
      (323.9844,500.1922) .. (323.5469,500.7963) .. controls (323.1146,501.3953) and
      (322.8984,502.2234) .. (322.8984,503.2807) .. controls (322.8984,504.3380) and
      (323.1146,505.1687) .. (323.5469,505.7729) .. controls (323.9844,506.3718) and
      (324.5833,506.6713) .. (325.3438,506.6713) .. controls (326.1042,506.6713) and
      (326.7005,506.3718) .. (327.1328,505.7729) .. controls (327.5703,505.1687) and
      (327.7891,504.3380) .. (327.7891,503.2807);

    % path5691
    \path[fill] (335.6406,503.2495) .. controls (334.4792,503.2495) and
      (333.6745,503.3823) .. (333.2266,503.6479) .. controls (332.7786,503.9135) and
      (332.5547,504.3666) .. (332.5547,505.0073) .. controls (332.5547,505.5177) and
      (332.7213,505.9239) .. (333.0547,506.2260) .. controls (333.3932,506.5229) and
      (333.8516,506.6713) .. (334.4297,506.6713) .. controls (335.2266,506.6713) and
      (335.8646,506.3901) .. (336.3438,505.8276) .. controls (336.8281,505.2599) and
      (337.0703,504.5073) .. (337.0703,503.5698) -- (337.0703,503.2495) --
      (335.6406,503.2495)(338.5078,502.6557) -- (338.5078,507.6479) --
      (337.0703,507.6479) -- (337.0703,506.3198) .. controls (336.7422,506.8510) and
      (336.3333,507.2442) .. (335.8438,507.4995) .. controls (335.3542,507.7495) and
      (334.7552,507.8745) .. (334.0469,507.8745) .. controls (333.1510,507.8745) and
      (332.4375,507.6245) .. (331.9062,507.1245) .. controls (331.3802,506.6192) and
      (331.1172,505.9448) .. (331.1172,505.1010) .. controls (331.1172,504.1166) and
      (331.4453,503.3745) .. (332.1016,502.8745) .. controls (332.7630,502.3745) and
      (333.7474,502.1245) .. (335.0547,502.1245) -- (337.0703,502.1245) --
      (337.0703,501.9838) .. controls (337.0703,501.3224) and (336.8516,500.8120) ..
      (336.4141,500.4526) .. controls (335.9818,500.0880) and (335.3724,499.9057) ..
      (334.5859,499.9057) .. controls (334.0859,499.9057) and (333.5990,499.9656) ..
      (333.1250,500.0854) .. controls (332.6510,500.2052) and (332.1953,500.3849) ..
      (331.7578,500.6245) -- (331.7578,499.2963) .. controls (332.2839,499.0932) and
      (332.7943,498.9422) .. (333.2891,498.8432) .. controls (333.7839,498.7390) and
      (334.2656,498.6870) .. (334.7344,498.6870) .. controls (336.0000,498.6870) and
      (336.9453,499.0151) .. (337.5703,499.6713) .. controls (338.1953,500.3276) and
      (338.5078,501.3224) .. (338.5078,502.6557);

    % path5693
    \path[fill] (340.8625,503.9838) -- (342.1940,503.9838) -- (344.4704,507.2577) --
      (346.7590,503.9838) -- (348.0905,503.9838) -- (345.1453,508.2133) --
      (348.2868,512.7908) -- (346.9554,512.7908) -- (344.3783,509.0450) --
      (341.7829,512.7908) -- (340.4453,512.7908) -- (343.7157,508.0894) --
      (340.8625,503.9838);

    % path5695
    \path[fill] (352.2188,495.5073) -- (353.4688,495.5073) .. controls
      (354.2500,496.7364) and (354.8333,497.9396) .. (355.2188,499.1166) .. controls
      (355.6094,500.2937) and (355.8047,501.4630) .. (355.8047,502.6244) .. controls
      (355.8047,503.7911) and (355.6094,504.9656) .. (355.2188,506.1479) .. controls
      (354.8333,507.3302) and (354.2500,508.5333) .. (353.4688,509.7573) --
      (352.2188,509.7573) .. controls (352.9115,508.5645) and (353.4271,507.3797) ..
      (353.7656,506.2026) .. controls (354.1094,505.0203) and (354.2812,503.8276) ..
      (354.2812,502.6244) .. controls (354.2812,501.4213) and (354.1094,500.2338) ..
      (353.7656,499.0619) .. controls (353.4271,497.8901) and (352.9115,496.7052) ..
      (352.2188,495.5073);

  \end{scope}
  % rect5727
  \path[draw=black,miter limit=4.00,line width=1.400pt] (263.9286,333.5050) --
    (362.5000,333.5050) -- (362.5000,373.5050) -- (263.9286,373.5050) -- cycle;

  % rect5727-0
  \path[draw=black,miter limit=4.00,line width=1.400pt] (263.9286,627.2193) --
    (362.5000,627.2193) -- (362.5000,667.2193) -- (263.9286,667.2193) -- cycle;

  \begin{scope}[shift={(-7.07143,590.0292)},fill=black]% flowRoot5729-9
    % path5700
    \path[fill] (284.5859,50.4870) -- (284.5859,54.8699) -- (286.5703,54.8699) ..
      controls (287.3047,54.8699) and (287.8724,54.6798) .. (288.2734,54.2995) ..
      controls (288.6745,53.9193) and (288.8750,53.3777) .. (288.8750,52.6745) ..
      controls (288.8750,51.9766) and (288.6745,51.4376) .. (288.2734,51.0574) ..
      controls (287.8724,50.6772) and (287.3047,50.4871) .. (286.5703,50.4870) --
      (284.5859,50.4870)(283.0078,49.1902) -- (286.5703,49.1902) .. controls
      (287.8776,49.1902) and (288.8646,49.4871) .. (289.5312,50.0808) .. controls
      (290.2031,50.6694) and (290.5390,51.5339) .. (290.5391,52.6745) .. controls
      (290.5391,53.8256) and (290.2031,54.6954) .. (289.5312,55.2839) .. controls
      (288.8646,55.8725) and (287.8776,56.1667) .. (286.5703,56.1667) --
      (284.5859,56.1667) -- (284.5859,60.8542) -- (283.0078,60.8542) --
      (283.0078,49.1902);

    % path5702
    \path[fill] (296.0547,48.7136) .. controls (295.3568,49.9115) and
      (294.8385,51.0964) .. (294.5000,52.2683) .. controls (294.1615,53.4402) and
      (293.9922,54.6277) .. (293.9922,55.8308) .. controls (293.9922,57.0339) and
      (294.1615,58.2266) .. (294.5000,59.4089) .. controls (294.8438,60.5860) and
      (295.3620,61.7709) .. (296.0547,62.9636) -- (294.8047,62.9636) .. controls
      (294.0234,61.7397) and (293.4375,60.5365) .. (293.0469,59.3542) .. controls
      (292.6615,58.1719) and (292.4688,56.9975) .. (292.4688,55.8308) .. controls
      (292.4688,54.6693) and (292.6615,53.5001) .. (293.0469,52.3230) .. controls
      (293.4323,51.1459) and (294.0182,49.9428) .. (294.8047,48.7136) --
      (296.0547,48.7136);

    % path5704
    \path[fill] (299.0547,58.8699) -- (300.7031,58.8699) -- (300.7031,60.8542) --
      (299.0547,60.8542) -- (299.0547,58.8699);

    % path5706
    \path[fill] (304.1484,58.8699) -- (305.7969,58.8699) -- (305.7969,60.8542) --
      (304.1484,60.8542) -- (304.1484,58.8699);

    % path5708
    \path[fill] (309.2422,58.8699) -- (310.8906,58.8699) -- (310.8906,60.8542) --
      (309.2422,60.8542) -- (309.2422,58.8699);

    % path5710
    \path[fill] (315.9844,48.6277) -- (315.9844,64.6277) -- (314.6562,64.6277) --
      (314.6562,48.6277) -- (315.9844,48.6277);

    % path5712
    \path[fill] (319.0391,49.1902) -- (320.7344,49.1902) -- (323.6328,53.5261) --
      (326.5469,49.1902) -- (328.2422,49.1902) -- (324.4922,54.7917) --
      (328.4922,60.8542) -- (326.7969,60.8542) -- (323.5156,55.8933) --
      (320.2109,60.8542) -- (318.5078,60.8542) -- (322.6719,54.6277) --
      (319.0391,49.1902);

    % path5714
    \path[fill] (330.8750,58.8699) -- (332.5234,58.8699) -- (332.5234,60.2136) --
      (331.2422,62.7136) -- (330.2344,62.7136) -- (330.8750,60.2136) --
      (330.8750,58.8699);

    % path5716
    \path[fill] (335.8047,58.8699) -- (337.4531,58.8699) -- (337.4531,60.8542) --
      (335.8047,60.8542) -- (335.8047,58.8699);

    % path5718
    \path[fill] (340.8984,58.8699) -- (342.5469,58.8699) -- (342.5469,60.8542) --
      (340.8984,60.8542) -- (340.8984,58.8699);

    % path5720
    \path[fill] (345.9922,58.8699) -- (347.6406,58.8699) -- (347.6406,60.8542) --
      (345.9922,60.8542) -- (345.9922,58.8699);

    % path5722
    \path[fill] (350.6562,48.7136) -- (351.9062,48.7136) .. controls
      (352.6875,49.9428) and (353.2708,51.1459) .. (353.6562,52.3230) .. controls
      (354.0469,53.5001) and (354.2422,54.6693) .. (354.2422,55.8308) .. controls
      (354.2422,56.9975) and (354.0469,58.1719) .. (353.6562,59.3542) .. controls
      (353.2708,60.5365) and (352.6875,61.7397) .. (351.9062,62.9636) --
      (350.6562,62.9636) .. controls (351.3490,61.7709) and (351.8646,60.5860) ..
      (352.2031,59.4089) .. controls (352.5469,58.2266) and (352.7188,57.0339) ..
      (352.7188,55.8308) .. controls (352.7188,54.6277) and (352.5469,53.4402) ..
      (352.2031,52.2683) .. controls (351.8646,51.0964) and (351.3490,49.9115) ..
      (350.6562,48.7136);

  \end{scope}
  % rect6565
  \path[draw=black,miter limit=4.00,line width=1.400pt] (398.5714,421.5050) --
    (479.2857,421.5050) -- (479.2857,459.3621) -- (398.5714,459.3621) -- cycle;

  \begin{scope}[fill=black]% text6567
    % path5726
    \path[fill] (417.4342,433.9950) -- (417.4342,438.3778) -- (419.4185,438.3778) ..
      controls (420.1529,438.3778) and (420.7206,438.1877) .. (421.1217,437.8075) ..
      controls (421.5227,437.4273) and (421.7232,436.8856) .. (421.7232,436.1825) ..
      controls (421.7232,435.4846) and (421.5227,434.9455) .. (421.1217,434.5653) ..
      controls (420.7206,434.1851) and (420.1529,433.9950) .. (419.4185,433.9950) --
      (417.4342,433.9950)(415.8560,432.6981) -- (419.4185,432.6981) .. controls
      (420.7258,432.6981) and (421.7128,432.9950) .. (422.3795,433.5888) .. controls
      (423.0513,434.1773) and (423.3873,435.0419) .. (423.3873,436.1825) .. controls
      (423.3873,437.3335) and (423.0513,438.2033) .. (422.3795,438.7919) .. controls
      (421.7128,439.3804) and (420.7258,439.6747) .. (419.4185,439.6747) --
      (417.4342,439.6747) -- (417.4342,444.3622) -- (415.8560,444.3622) --
      (415.8560,432.6981);

    % path5728
    \path[fill] (428.9029,432.2216) .. controls (428.2050,433.4195) and
      (427.6868,434.6044) .. (427.3482,435.7763) .. controls (427.0097,436.9481) and
      (426.8404,438.1356) .. (426.8404,439.3388) .. controls (426.8404,440.5419) and
      (427.0097,441.7346) .. (427.3482,442.9169) .. controls (427.6920,444.0940) and
      (428.2102,445.2789) .. (428.9029,446.4716) -- (427.6529,446.4716) .. controls
      (426.8717,445.2476) and (426.2857,444.0445) .. (425.8951,442.8622) .. controls
      (425.5097,441.6799) and (425.3170,440.5054) .. (425.3170,439.3388) .. controls
      (425.3170,438.1773) and (425.5097,437.0080) .. (425.8951,435.8309) .. controls
      (426.2805,434.6539) and (426.8665,433.4507) .. (427.6529,432.2216) --
      (428.9029,432.2216);

    % path5730
    \path[fill] (433.7310,437.4872) -- (439.2467,437.4872) -- (439.2467,438.8153) --
      (433.7310,438.8153) -- (433.7310,437.4872)(436.4967,433.7684) .. controls
      (435.3508,433.7684) and (434.4420,434.1955) .. (433.7701,435.0497) .. controls
      (433.0930,435.9039) and (432.7545,437.0679) .. (432.7545,438.5419) .. controls
      (432.7545,440.0106) and (433.0904,441.1721) .. (433.7623,442.0263) .. controls
      (434.4394,442.8804) and (435.3508,443.3075) .. (436.4967,443.3075) .. controls
      (437.6425,443.3075) and (438.5514,442.8804) .. (439.2232,442.0263) .. controls
      (439.8899,441.1721) and (440.2232,440.0106) .. (440.2232,438.5419) .. controls
      (440.2232,437.0679) and (439.8899,435.9039) .. (439.2232,435.0497) .. controls
      (438.5513,434.1955) and (437.6425,433.7684) ..
      (436.4967,433.7684)(436.4967,432.4872) .. controls (438.1321,432.4872) and
      (439.4394,433.0367) .. (440.4185,434.1356) .. controls (441.3977,435.2294) and
      (441.8873,436.6981) .. (441.8873,438.5419) .. controls (441.8873,440.3804) and
      (441.3977,441.8466) .. (440.4185,442.9403) .. controls (439.4394,444.0393) and
      (438.1321,444.5887) .. (436.4967,444.5888) .. controls (434.8560,444.5887) and
      (433.5462,444.0419) .. (432.5670,442.9481) .. controls (431.5826,441.8544) and
      (431.0904,440.3856) .. (431.0904,438.5419) .. controls (431.0904,436.6981) and
      (431.5826,435.2268) .. (432.5670,434.1278) .. controls (433.5461,433.0341) and
      (434.8560,432.4872) .. (436.4967,432.4872);

    % path5732
    \path[fill] (444.0982,432.2216) -- (445.3482,432.2216) .. controls
      (446.1295,433.4507) and (446.7128,434.6539) .. (447.0982,435.8309) .. controls
      (447.4889,437.0080) and (447.6842,438.1773) .. (447.6842,439.3388) .. controls
      (447.6842,440.5054) and (447.4889,441.6799) .. (447.0982,442.8622) .. controls
      (446.7128,444.0445) and (446.1295,445.2476) .. (445.3482,446.4716) --
      (444.0982,446.4716) .. controls (444.7909,445.2789) and (445.3066,444.0940) ..
      (445.6451,442.9169) .. controls (445.9889,441.7346) and (446.1607,440.5419) ..
      (446.1607,439.3388) .. controls (446.1607,438.1356) and (445.9889,436.9481) ..
      (445.6451,435.7763) .. controls (445.3066,434.6044) and (444.7909,433.4195) ..
      (444.0982,432.2216);

  \end{scope}
  % rect5727-2
  \path[draw=black,miter limit=4.00,line width=1.041pt] (783.5335,337.6479) --
    (854.8282,337.6479) -- (854.8282,377.6479) -- (783.5335,377.6479) -- cycle;

  \begin{scope}[shift={(508.53349,300.45777)},fill=black]% flowRoot5729-94
    % path5736
    \path[fill] (287.7422,50.2605) .. controls (286.5963,50.2605) and
      (285.6849,50.6876) .. (285.0078,51.5417) .. controls (284.3359,52.3959) and
      (284.0000,53.5600) .. (284.0000,55.0339) .. controls (284.0000,56.5027) and
      (284.3359,57.6641) .. (285.0078,58.5183) .. controls (285.6849,59.3725) and
      (286.5964,59.7995) .. (287.7422,59.7995) .. controls (288.8880,59.7995) and
      (289.7943,59.3725) .. (290.4609,58.5183) .. controls (291.1328,57.6641) and
      (291.4687,56.5027) .. (291.4688,55.0339) .. controls (291.4687,53.5600) and
      (291.1328,52.3959) .. (290.4609,51.5417) .. controls (289.7943,50.6876) and
      (288.8880,50.2605) .. (287.7422,50.2605)(289.9531,60.6433) --
      (292.0312,62.9167) -- (290.1250,62.9167) -- (288.3984,61.0495) .. controls
      (288.2266,61.0600) and (288.0937,61.0678) .. (288.0000,61.0730) .. controls
      (287.9115,61.0782) and (287.8255,61.0808) .. (287.7422,61.0808) .. controls
      (286.1016,61.0808) and (284.7891,60.5339) .. (283.8047,59.4402) .. controls
      (282.8255,58.3412) and (282.3359,56.8725) .. (282.3359,55.0339) .. controls
      (282.3359,53.1902) and (282.8255,51.7214) .. (283.8047,50.6277) .. controls
      (284.7891,49.5287) and (286.1016,48.9792) .. (287.7422,48.9792) .. controls
      (289.3776,48.9792) and (290.6849,49.5287) .. (291.6641,50.6277) .. controls
      (292.6432,51.7214) and (293.1328,53.1902) .. (293.1328,55.0339) .. controls
      (293.1328,56.3881) and (292.8594,57.5469) .. (292.3125,58.5105) .. controls
      (291.7708,59.4740) and (290.9844,60.1850) .. (289.9531,60.6433);

    % path5738
    \path[fill] (298.9922,48.7136) .. controls (298.2943,49.9115) and
      (297.7760,51.0964) .. (297.4375,52.2683) .. controls (297.0990,53.4402) and
      (296.9297,54.6277) .. (296.9297,55.8308) .. controls (296.9297,57.0339) and
      (297.0990,58.2266) .. (297.4375,59.4089) .. controls (297.7812,60.5860) and
      (298.2995,61.7709) .. (298.9922,62.9636) -- (297.7422,62.9636) .. controls
      (296.9609,61.7397) and (296.3750,60.5365) .. (295.9844,59.3542) .. controls
      (295.5990,58.1719) and (295.4062,56.9975) .. (295.4062,55.8308) .. controls
      (295.4062,54.6693) and (295.5990,53.5001) .. (295.9844,52.3230) .. controls
      (296.3698,51.1459) and (296.9557,49.9428) .. (297.7422,48.7136) --
      (298.9922,48.7136);

    % path5740
    \path[fill] (303.1797,59.5417) -- (303.1797,64.1824) -- (301.7344,64.1824) --
      (301.7344,52.1042) -- (303.1797,52.1042) -- (303.1797,53.4324) .. controls
      (303.4818,52.9115) and (303.8620,52.5261) .. (304.3203,52.2761) .. controls
      (304.7839,52.0209) and (305.3359,51.8933) .. (305.9766,51.8933) .. controls
      (307.0391,51.8933) and (307.9010,52.3152) .. (308.5625,53.1589) .. controls
      (309.2292,54.0027) and (309.5625,55.1121) .. (309.5625,56.4870) .. controls
      (309.5625,57.8621) and (309.2292,58.9714) .. (308.5625,59.8152) .. controls
      (307.9010,60.6589) and (307.0391,61.0808) .. (305.9766,61.0808) .. controls
      (305.3359,61.0808) and (304.7839,60.9558) .. (304.3203,60.7058) .. controls
      (303.8620,60.4506) and (303.4818,60.0626) ..
      (303.1797,59.5417)(308.0703,56.4870) .. controls (308.0703,55.4298) and
      (307.8516,54.6016) .. (307.4141,54.0027) .. controls (306.9818,53.3985) and
      (306.3854,53.0964) .. (305.6250,53.0964) .. controls (304.8646,53.0964) and
      (304.2656,53.3985) .. (303.8281,54.0027) .. controls (303.3958,54.6016) and
      (303.1797,55.4298) .. (303.1797,56.4870) .. controls (303.1797,57.5443) and
      (303.3958,58.3751) .. (303.8281,58.9792) .. controls (304.2656,59.5782) and
      (304.8646,59.8777) .. (305.6250,59.8777) .. controls (306.3854,59.8777) and
      (306.9818,59.5782) .. (307.4141,58.9792) .. controls (307.8516,58.3751) and
      (308.0703,57.5443) .. (308.0703,56.4870);

    % path5742
    \path[fill] (315.9219,56.4558) .. controls (314.7604,56.4558) and
      (313.9557,56.5886) .. (313.5078,56.8542) .. controls (313.0599,57.1199) and
      (312.8359,57.5730) .. (312.8359,58.2136) .. controls (312.8359,58.7240) and
      (313.0026,59.1303) .. (313.3359,59.4324) .. controls (313.6745,59.7292) and
      (314.1328,59.8777) .. (314.7109,59.8777) .. controls (315.5078,59.8777) and
      (316.1458,59.5964) .. (316.6250,59.0339) .. controls (317.1094,58.4662) and
      (317.3516,57.7136) .. (317.3516,56.7761) -- (317.3516,56.4558) --
      (315.9219,56.4558)(318.7891,55.8620) -- (318.7891,60.8542) --
      (317.3516,60.8542) -- (317.3516,59.5261) .. controls (317.0234,60.0574) and
      (316.6146,60.4506) .. (316.1250,60.7058) .. controls (315.6354,60.9558) and
      (315.0365,61.0808) .. (314.3281,61.0808) .. controls (313.4323,61.0808) and
      (312.7188,60.8308) .. (312.1875,60.3308) .. controls (311.6615,59.8256) and
      (311.3984,59.1511) .. (311.3984,58.3074) .. controls (311.3984,57.3230) and
      (311.7266,56.5808) .. (312.3828,56.0808) .. controls (313.0443,55.5808) and
      (314.0286,55.3308) .. (315.3359,55.3308) -- (317.3516,55.3308) --
      (317.3516,55.1902) .. controls (317.3516,54.5287) and (317.1328,54.0183) ..
      (316.6953,53.6589) .. controls (316.2630,53.2943) and (315.6536,53.1121) ..
      (314.8672,53.1120) .. controls (314.3672,53.1121) and (313.8802,53.1719) ..
      (313.4062,53.2917) .. controls (312.9323,53.4115) and (312.4766,53.5912) ..
      (312.0391,53.8308) -- (312.0391,52.5027) .. controls (312.5651,52.2996) and
      (313.0755,52.1485) .. (313.5703,52.0495) .. controls (314.0651,51.9454) and
      (314.5469,51.8933) .. (315.0156,51.8933) .. controls (316.2812,51.8933) and
      (317.2266,52.2214) .. (317.8516,52.8777) .. controls (318.4766,53.5339) and
      (318.7891,54.5287) .. (318.7891,55.8620);

    % path5744
    \path[fill] (320.9051,56.4725) -- (322.0071,56.4725) -- (323.8911,59.2909) --
      (325.7852,56.4725) -- (326.8872,56.4725) -- (324.4497,60.1136) --
      (327.0497,64.0542) -- (325.9477,64.0542) -- (323.8149,60.8296) --
      (321.6668,64.0542) -- (320.5598,64.0542) -- (323.2664,60.0069) --
      (320.9051,56.4725);

    % path5746
    \path[fill] (328.6610,48.7136) -- (329.9110,48.7136) .. controls
      (330.6923,49.9428) and (331.2756,51.1459) .. (331.6610,52.3230) .. controls
      (332.0517,53.5001) and (332.2470,54.6693) .. (332.2470,55.8308) .. controls
      (332.2470,56.9975) and (332.0517,58.1719) .. (331.6610,59.3542) .. controls
      (331.2756,60.5365) and (330.6923,61.7397) .. (329.9110,62.9636) --
      (328.6610,62.9636) .. controls (329.3537,61.7709) and (329.8694,60.5860) ..
      (330.2079,59.4089) .. controls (330.5517,58.2266) and (330.7235,57.0339) ..
      (330.7235,55.8308) .. controls (330.7235,54.6277) and (330.5517,53.4402) ..
      (330.2079,52.2683) .. controls (329.8694,51.0964) and (329.3537,49.9115) ..
      (328.6610,48.7136);

  \end{scope}
  % rect5727-2-6
  \path[draw=black,miter limit=4.00,line width=1.041pt] (783.5335,495.5050) --
    (854.8282,495.5050) -- (854.8282,535.5050) -- (783.5335,535.5050) -- cycle;

  \begin{scope}[shift={(519.96206,458.3149)},fill=black]% flowRoot5729-94-0
    % path5750
    \path[fill] (287.7422,50.2605) .. controls (286.5963,50.2605) and
      (285.6849,50.6876) .. (285.0078,51.5417) .. controls (284.3359,52.3959) and
      (284.0000,53.5600) .. (284.0000,55.0339) .. controls (284.0000,56.5027) and
      (284.3359,57.6641) .. (285.0078,58.5183) .. controls (285.6849,59.3725) and
      (286.5964,59.7995) .. (287.7422,59.7995) .. controls (288.8880,59.7995) and
      (289.7943,59.3725) .. (290.4609,58.5183) .. controls (291.1328,57.6641) and
      (291.4687,56.5027) .. (291.4688,55.0339) .. controls (291.4687,53.5600) and
      (291.1328,52.3959) .. (290.4609,51.5417) .. controls (289.7943,50.6876) and
      (288.8880,50.2605) .. (287.7422,50.2605)(289.9531,60.6433) --
      (292.0312,62.9167) -- (290.1250,62.9167) -- (288.3984,61.0495) .. controls
      (288.2266,61.0600) and (288.0937,61.0678) .. (288.0000,61.0730) .. controls
      (287.9115,61.0782) and (287.8255,61.0808) .. (287.7422,61.0808) .. controls
      (286.1016,61.0808) and (284.7891,60.5339) .. (283.8047,59.4402) .. controls
      (282.8255,58.3412) and (282.3359,56.8725) .. (282.3359,55.0339) .. controls
      (282.3359,53.1902) and (282.8255,51.7214) .. (283.8047,50.6277) .. controls
      (284.7891,49.5287) and (286.1016,48.9792) .. (287.7422,48.9792) .. controls
      (289.3776,48.9792) and (290.6849,49.5287) .. (291.6641,50.6277) .. controls
      (292.6432,51.7214) and (293.1328,53.1902) .. (293.1328,55.0339) .. controls
      (293.1328,56.3881) and (292.8594,57.5469) .. (292.3125,58.5105) .. controls
      (291.7708,59.4740) and (290.9844,60.1850) .. (289.9531,60.6433);

    % path5752
    \path[fill] (298.9922,48.7136) .. controls (298.2943,49.9115) and
      (297.7760,51.0964) .. (297.4375,52.2683) .. controls (297.0990,53.4402) and
      (296.9297,54.6277) .. (296.9297,55.8308) .. controls (296.9297,57.0339) and
      (297.0990,58.2266) .. (297.4375,59.4089) .. controls (297.7812,60.5860) and
      (298.2995,61.7709) .. (298.9922,62.9636) -- (297.7422,62.9636) .. controls
      (296.9609,61.7397) and (296.3750,60.5365) .. (295.9844,59.3542) .. controls
      (295.5990,58.1719) and (295.4062,56.9975) .. (295.4062,55.8308) .. controls
      (295.4062,54.6693) and (295.5990,53.5001) .. (295.9844,52.3230) .. controls
      (296.3698,51.1459) and (296.9557,49.9428) .. (297.7422,48.7136) --
      (298.9922,48.7136);

    % path5754
    \path[fill] (301.2891,49.1902) -- (302.9844,49.1902) -- (305.8828,53.5261) --
      (308.7969,49.1902) -- (310.4922,49.1902) -- (306.7422,54.7917) --
      (310.7422,60.8542) -- (309.0469,60.8542) -- (305.7656,55.8933) --
      (302.4609,60.8542) -- (300.7578,60.8542) -- (304.9219,54.6277) --
      (301.2891,49.1902);

    % path5756
    \path[fill] (312.5312,48.7136) -- (313.7812,48.7136) .. controls
      (314.5625,49.9428) and (315.1458,51.1459) .. (315.5312,52.3230) .. controls
      (315.9219,53.5001) and (316.1172,54.6693) .. (316.1172,55.8308) .. controls
      (316.1172,56.9975) and (315.9219,58.1719) .. (315.5312,59.3542) .. controls
      (315.1458,60.5365) and (314.5625,61.7397) .. (313.7812,62.9636) --
      (312.5312,62.9636) .. controls (313.2240,61.7709) and (313.7396,60.5860) ..
      (314.0781,59.4089) .. controls (314.4219,58.2266) and (314.5938,57.0339) ..
      (314.5938,55.8308) .. controls (314.5938,54.6277) and (314.4219,53.4402) ..
      (314.0781,52.2683) .. controls (313.7396,51.0964) and (313.2240,49.9115) ..
      (312.5312,48.7136);

  \end{scope}
  % rect5727-2-6-2
  \path[draw=black,miter limit=4.00,line width=1.041pt] (882.3527,406.3621) --
    (953.6473,406.3621) -- (953.6473,446.3621) -- (882.3527,446.3621) -- cycle;

  \begin{scope}[shift={(613.78125,369.17204)},fill=black]% flowRoot5729-94-0-6
    % path5760
    \path[fill] (287.7422,50.2605) .. controls (286.5963,50.2605) and
      (285.6849,50.6876) .. (285.0078,51.5417) .. controls (284.3359,52.3959) and
      (284.0000,53.5600) .. (284.0000,55.0339) .. controls (284.0000,56.5027) and
      (284.3359,57.6641) .. (285.0078,58.5183) .. controls (285.6849,59.3725) and
      (286.5964,59.7995) .. (287.7422,59.7995) .. controls (288.8880,59.7995) and
      (289.7943,59.3725) .. (290.4609,58.5183) .. controls (291.1328,57.6641) and
      (291.4687,56.5027) .. (291.4688,55.0339) .. controls (291.4687,53.5600) and
      (291.1328,52.3959) .. (290.4609,51.5417) .. controls (289.7943,50.6876) and
      (288.8880,50.2605) .. (287.7422,50.2605)(289.9531,60.6433) --
      (292.0312,62.9167) -- (290.1250,62.9167) -- (288.3984,61.0495) .. controls
      (288.2266,61.0600) and (288.0937,61.0678) .. (288.0000,61.0730) .. controls
      (287.9115,61.0782) and (287.8255,61.0808) .. (287.7422,61.0808) .. controls
      (286.1016,61.0808) and (284.7891,60.5339) .. (283.8047,59.4402) .. controls
      (282.8255,58.3412) and (282.3359,56.8725) .. (282.3359,55.0339) .. controls
      (282.3359,53.1902) and (282.8255,51.7214) .. (283.8047,50.6277) .. controls
      (284.7891,49.5287) and (286.1016,48.9792) .. (287.7422,48.9792) .. controls
      (289.3776,48.9792) and (290.6849,49.5287) .. (291.6641,50.6277) .. controls
      (292.6432,51.7214) and (293.1328,53.1902) .. (293.1328,55.0339) .. controls
      (293.1328,56.3881) and (292.8594,57.5469) .. (292.3125,58.5105) .. controls
      (291.7708,59.4740) and (290.9844,60.1850) .. (289.9531,60.6433);

    % path5762
    \path[fill] (298.9922,48.7136) .. controls (298.2943,49.9115) and
      (297.7760,51.0964) .. (297.4375,52.2683) .. controls (297.0990,53.4402) and
      (296.9297,54.6277) .. (296.9297,55.8308) .. controls (296.9297,57.0339) and
      (297.0990,58.2266) .. (297.4375,59.4089) .. controls (297.7812,60.5860) and
      (298.2995,61.7709) .. (298.9922,62.9636) -- (297.7422,62.9636) .. controls
      (296.9609,61.7397) and (296.3750,60.5365) .. (295.9844,59.3542) .. controls
      (295.5990,58.1719) and (295.4062,56.9975) .. (295.4062,55.8308) .. controls
      (295.4062,54.6693) and (295.5990,53.5001) .. (295.9844,52.3230) .. controls
      (296.3698,51.1459) and (296.9557,49.9428) .. (297.7422,48.7136) --
      (298.9922,48.7136);

    % path5764
    \path[fill] (303.8203,53.9792) -- (309.3359,53.9792) -- (309.3359,55.3074) --
      (303.8203,55.3074) -- (303.8203,53.9792)(306.5859,50.2605) .. controls
      (305.4401,50.2605) and (304.5312,50.6876) .. (303.8594,51.5417) .. controls
      (303.1823,52.3959) and (302.8438,53.5600) .. (302.8438,55.0339) .. controls
      (302.8438,56.5027) and (303.1797,57.6641) .. (303.8516,58.5183) .. controls
      (304.5286,59.3725) and (305.4401,59.7995) .. (306.5859,59.7995) .. controls
      (307.7318,59.7995) and (308.6406,59.3725) .. (309.3125,58.5183) .. controls
      (309.9792,57.6641) and (310.3125,56.5027) .. (310.3125,55.0339) .. controls
      (310.3125,53.5600) and (309.9792,52.3959) .. (309.3125,51.5417) .. controls
      (308.6406,50.6876) and (307.7318,50.2605) ..
      (306.5859,50.2605)(306.5859,48.9792) .. controls (308.2213,48.9792) and
      (309.5286,49.5287) .. (310.5078,50.6277) .. controls (311.4870,51.7214) and
      (311.9766,53.1902) .. (311.9766,55.0339) .. controls (311.9766,56.8725) and
      (311.4870,58.3386) .. (310.5078,59.4324) .. controls (309.5286,60.5313) and
      (308.2214,61.0808) .. (306.5859,61.0808) .. controls (304.9453,61.0808) and
      (303.6354,60.5339) .. (302.6562,59.4402) .. controls (301.6719,58.3464) and
      (301.1797,56.8777) .. (301.1797,55.0339) .. controls (301.1797,53.1902) and
      (301.6719,51.7188) .. (302.6562,50.6199) .. controls (303.6354,49.5261) and
      (304.9453,48.9792) .. (306.5859,48.9792);

    % path5766
    \path[fill] (314.1875,48.7136) -- (315.4375,48.7136) .. controls
      (316.2188,49.9428) and (316.8021,51.1459) .. (317.1875,52.3230) .. controls
      (317.5781,53.5001) and (317.7734,54.6693) .. (317.7734,55.8308) .. controls
      (317.7734,56.9975) and (317.5781,58.1719) .. (317.1875,59.3542) .. controls
      (316.8021,60.5365) and (316.2188,61.7397) .. (315.4375,62.9636) --
      (314.1875,62.9636) .. controls (314.8802,61.7709) and (315.3958,60.5860) ..
      (315.7344,59.4089) .. controls (316.0781,58.2266) and (316.2500,57.0339) ..
      (316.2500,55.8308) .. controls (316.2500,54.6277) and (316.0781,53.4402) ..
      (315.7344,52.2683) .. controls (315.3958,51.0964) and (314.8802,49.9115) ..
      (314.1875,48.7136);

  \end{scope}
  % path2985-07
  \path[shift={(493.35715,363.09559)},draw=black,miter limit=4.00,line
    width=1.640pt] (87.1429,64.0523) .. controls (87.1429,78.4511) and
    (75.6302,90.1237) .. (61.4286,90.1237) .. controls (47.2270,90.1237) and
    (35.7143,78.4511) .. (35.7143,64.0523) .. controls (35.7143,49.6534) and
    (47.2270,37.9809) .. (61.4286,37.9809) .. controls (75.6302,37.9809) and
    (87.1429,49.6534) .. (87.1429,64.0523) -- cycle;

  \begin{scope}[shift={(504.78572,-198.07143)}]% g3771-6-7
    \begin{scope}[shift={(-10.0,190.0)}]% g3811-28
      % path2985-0-4
      \path[shift={(0,527.95273)},draw=black,miter limit=4.00,line width=1.640pt]
        (87.1429,64.0523) .. controls (87.1429,78.4511) and (75.6302,90.1237) ..
        (61.4286,90.1237) .. controls (47.2270,90.1237) and (35.7143,78.4511) ..
        (35.7143,64.0523) .. controls (35.7143,49.6534) and (47.2270,37.9809) ..
        (61.4286,37.9809) .. controls (75.6302,37.9809) and (87.1429,49.6534) ..
        (87.1429,64.0523) -- cycle;

      \begin{scope}[shift={(6.42857,531.52416)},fill=black]% flowRoot3763-8-0
        % path5771
        \path[fill] (49.5703,53.6902) -- (51.2656,53.6902) -- (54.1641,58.0261) --
          (57.0781,53.6902) -- (58.7734,53.6902) -- (55.0234,59.2917) --
          (59.0234,65.3542) -- (57.3281,65.3542) -- (54.0469,60.3933) --
          (50.7422,65.3542) -- (49.0391,65.3542) -- (53.2031,59.1277) --
          (49.5703,53.6902);

      \end{scope}
    \end{scope}
  \end{scope}
  % path2985-6-1
  \path[shift={(757.75224,361.30987)},draw=black,miter limit=4.00,line
    width=1.640pt] (87.1429,64.0523) .. controls (87.1429,78.4511) and
    (75.6302,90.1237) .. (61.4286,90.1237) .. controls (47.2270,90.1237) and
    (35.7143,78.4511) .. (35.7143,64.0523) .. controls (35.7143,49.6534) and
    (47.2270,37.9809) .. (61.4286,37.9809) .. controls (75.6302,37.9809) and
    (87.1429,49.6534) .. (87.1429,64.0523) -- cycle;

  \begin{scope}[shift={(757.75224,-13.07143)}]% g3811-24-6
    % path2985-0-6-0
    \path[shift={(0,527.95273)},draw=black,miter limit=4.00,line width=1.640pt]
      (87.1429,64.0523) .. controls (87.1429,78.4511) and (75.6302,90.1237) ..
      (61.4286,90.1237) .. controls (47.2270,90.1237) and (35.7143,78.4511) ..
      (35.7143,64.0523) .. controls (35.7143,49.6534) and (47.2270,37.9809) ..
      (61.4286,37.9809) .. controls (75.6302,37.9809) and (87.1429,49.6534) ..
      (87.1429,64.0523) -- cycle;

    \begin{scope}[shift={(6.42857,531.52416)},fill=black]% flowRoot3763-8-8-6
      % path5776
      \path[fill] (49.5703,53.6902) -- (51.2656,53.6902) -- (54.1641,58.0261) --
        (57.0781,53.6902) -- (58.7734,53.6902) -- (55.0234,59.2917) --
        (59.0234,65.3542) -- (57.3281,65.3542) -- (54.0469,60.3933) --
        (50.7422,65.3542) -- (49.0391,65.3542) -- (53.2031,59.1277) --
        (49.5703,53.6902);

    \end{scope}
  \end{scope}
  % path2985-07-7
  \path[shift={(601.28572,431.45272)},draw=black,miter limit=4.00,line
    width=1.640pt] (87.1429,64.0523) .. controls (87.1429,78.4511) and
    (75.6302,90.1237) .. (61.4286,90.1237) .. controls (47.2270,90.1237) and
    (35.7143,78.4511) .. (35.7143,64.0523) .. controls (35.7143,49.6534) and
    (47.2270,37.9809) .. (61.4286,37.9809) .. controls (75.6302,37.9809) and
    (87.1429,49.6534) .. (87.1429,64.0523) -- cycle;

  % path2985-07-7-9
  \path[shift={(856.57143,429.30987)},draw=black,miter limit=4.00,line
    width=1.640pt] (87.1429,64.0523) .. controls (87.1429,78.4511) and
    (75.6302,90.1237) .. (61.4286,90.1237) .. controls (47.2270,90.1237) and
    (35.7143,78.4511) .. (35.7143,64.0523) .. controls (35.7143,49.6534) and
    (47.2270,37.9809) .. (61.4286,37.9809) .. controls (75.6302,37.9809) and
    (87.1429,49.6534) .. (87.1429,64.0523) -- cycle;

  % path6830
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.400pt] (918.0000,446.3621) -- (918.0000,467.2907);

  % path6834
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.400pt] (313.2143,373.5050) -- (313.2143,400.1479);

  % path6836
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.400pt] (313.5491,452.2886) -- (313.9390,482.6479);

  % path6838
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.400pt] (313.9312,525.5050) -- (313.5586,553.7216);

  % path6840
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.400pt] (313.2143,605.8622) -- (313.2143,627.2193);

  % path6842
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.400pt] (438.9286,459.3621) -- (438.9286,477.1479);

  % path6844
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.400pt] (413.2149,503.3960) -- (366.0000,503.7205);

  \begin{scope}[shift={(0,312.37839)},fill=black]% flowRoot6846
    % path5781
    \path[fill] (18.9922,14.3608) -- (16.3164,21.6166) -- (21.6777,21.6166) --
      (18.9922,14.3608)(17.8789,12.4174) -- (20.1152,12.4174) -- (25.6719,26.9975)
      -- (23.6211,26.9975) -- (22.2930,23.2572) -- (15.7207,23.2572) --
      (14.3926,26.9975) -- (12.3125,26.9975) -- (17.8789,12.4174);

  \end{scope}
  \begin{scope}[shift={(-8.0,307.37839)},fill=black]% flowRoot6854
    % path5784
    \path[fill] (233.9356,25.0346) -- (233.9356,30.3764) -- (237.0996,30.3764) ..
      controls (238.1608,30.3764) and (238.9453,30.1583) .. (239.4531,29.7221) ..
      controls (239.9674,29.2794) and (240.2246,28.6056) .. (240.2246,27.7006) ..
      controls (240.2246,26.7892) and (239.9674,26.1186) .. (239.4531,25.6889) ..
      controls (238.9453,25.2527) and (238.1608,25.0346) .. (237.0996,25.0346) --
      (233.9356,25.0346)(233.9356,19.0385) -- (233.9356,23.4330) --
      (236.8555,23.4330) .. controls (237.8190,23.4330) and (238.5351,23.2540) ..
      (239.0039,22.8959) .. controls (239.4792,22.5313) and (239.7168,21.9780) ..
      (239.7168,21.2358) .. controls (239.7168,20.5001) and (239.4792,19.9500) ..
      (239.0039,19.5854) .. controls (238.5351,19.2208) and (237.8190,19.0385) ..
      (236.8555,19.0385) -- (233.9356,19.0385)(231.9629,17.4174) --
      (237.0020,17.4174) .. controls (238.5059,17.4174) and (239.6647,17.7299) ..
      (240.4785,18.3549) .. controls (241.2923,18.9799) and (241.6992,19.8686) ..
      (241.6992,21.0209) .. controls (241.6992,21.9129) and (241.4909,22.6225) ..
      (241.0742,23.1498) .. controls (240.6575,23.6772) and (240.0456,24.0060) ..
      (239.2383,24.1362) .. controls (240.2083,24.3445) and (240.9603,24.7807) ..
      (241.4941,25.4447) .. controls (242.0345,26.1023) and (242.3047,26.9259) ..
      (242.3047,27.9155) .. controls (242.3047,29.2175) and (241.8620,30.2234) ..
      (240.9766,30.9330) .. controls (240.0911,31.6427) and (238.8314,31.9975) ..
      (237.1973,31.9975) -- (231.9629,31.9975) -- (231.9629,17.4174);

  \end{scope}
  \begin{scope}[shift={(0,309.52487)},fill=black]% flowRoot6862
    % path5787
    \path[fill] (511.4434,16.3842) -- (511.4434,18.4643) .. controls
      (510.7793,17.8458) and (510.0696,17.3836) .. (509.3144,17.0776) .. controls
      (508.5657,16.7716) and (507.7682,16.6186) .. (506.9219,16.6186) .. controls
      (505.2552,16.6186) and (503.9792,17.1297) .. (503.0937,18.1518) .. controls
      (502.2083,19.1674) and (501.7656,20.6388) .. (501.7656,22.5658) .. controls
      (501.7656,24.4864) and (502.2083,25.9578) .. (503.0937,26.9799) .. controls
      (503.9792,27.9955) and (505.2552,28.5033) .. (506.9219,28.5033) .. controls
      (507.7682,28.5033) and (508.5657,28.3503) .. (509.3144,28.0444) .. controls
      (510.0696,27.7384) and (510.7793,27.2761) .. (511.4434,26.6576) --
      (511.4434,28.7182) .. controls (510.7532,29.1869) and (510.0208,29.5385) ..
      (509.2461,29.7729) .. controls (508.4778,30.0072) and (507.6640,30.1244) ..
      (506.8047,30.1244) .. controls (504.5976,30.1244) and (502.8594,29.4506) ..
      (501.5898,28.1030) .. controls (500.3203,26.7488) and (499.6855,24.9031) ..
      (499.6855,22.5658) .. controls (499.6855,20.2221) and (500.3203,18.3764) ..
      (501.5898,17.0287) .. controls (502.8594,15.6746) and (504.5976,14.9975) ..
      (506.8047,14.9975) .. controls (507.6771,14.9975) and (508.4974,15.1147) ..
      (509.2656,15.3490) .. controls (510.0403,15.5769) and (510.7663,15.9220) ..
      (511.4434,16.3842);

  \end{scope}
  \begin{scope}[shift={(236.57841,309.53464)},fill=black]% flowRoot6862-5
    % path5790
    \path[fill] (502.4980,16.8822) -- (502.4980,28.2201) -- (504.8809,28.2201) ..
      controls (506.8926,28.2201) and (508.3639,27.7644) .. (509.2949,26.8530) ..
      controls (510.2324,25.9415) and (510.7012,24.5027) .. (510.7012,22.5365) ..
      controls (510.7012,20.5834) and (510.2324,19.1544) .. (509.2949,18.2494) ..
      controls (508.3639,17.3380) and (506.8926,16.8823) .. (504.8809,16.8822) --
      (502.4980,16.8822)(500.5254,15.2612) -- (504.5781,15.2612) .. controls
      (507.4036,15.2612) and (509.4772,15.8504) .. (510.7988,17.0287) .. controls
      (512.1204,18.2006) and (512.7812,20.0366) .. (512.7812,22.5365) .. controls
      (512.7812,25.0496) and (512.1172,26.8953) .. (510.7891,28.0737) .. controls
      (509.4609,29.2520) and (507.3906,29.8412) .. (504.5781,29.8412) --
      (500.5254,29.8412) -- (500.5254,15.2612);

  \end{scope}
  % path6897
  \path[draw=black,line join=miter,line cap=butt,line width=0.800pt]
    (193.5714,308.7907) -- (193.5714,681.6478);

  % path6899
  \path[draw=black,line join=miter,line cap=butt,line width=0.800pt]
    (490.0000,309.5050) -- (490.0000,682.3622);

  % path6901
  \path[draw=black,line join=miter,line cap=butt,line width=0.800pt]
    (720.7143,308.0764) -- (720.7143,680.9336);

  % path7229
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.400pt] (819.1808,377.6479) -- (819.1808,399.2907);

  % path7231
  \path[draw=black,line join=miter,line cap=butt,miter limit=4.00,line
    width=1.400pt] (819.1808,535.5050) -- (819.1808,552.8622);

  \begin{scope}[shift={(380.93693,436.28923)},fill=black]% flowRoot3763-8-1-6
    % path5441-6
    \path[fill] (57.6345,73.0683) .. controls (59.6025,73.0683) and
      (61.3785,72.4443) .. (62.7705,71.2923) .. controls (65.0985,69.3723) and
      (65.7225,66.6603) .. (65.7225,64.5963) .. controls (65.7225,62.6283) and
      (65.1225,59.8923) .. (62.7705,57.9963) .. controls (61.3785,56.8683) and
      (59.5545,56.2203) .. (57.6345,56.2203) .. controls (55.7145,56.2203) and
      (53.8905,56.8683) .. (52.4985,57.9963) .. controls (50.1465,59.8923) and
      (49.5465,62.6283) .. (49.5465,64.5963) .. controls (49.5465,66.6603) and
      (50.1705,69.3723) .. (52.4985,71.2923) .. controls (53.8905,72.4443) and
      (55.6665,73.0683) .. (57.6345,73.0683)(53.6025,59.3643) .. controls
      (54.4665,57.9243) and (56.0505,57.1563) .. (57.6345,57.1563) .. controls
      (59.2185,57.1563) and (60.8025,57.9243) .. (61.6665,59.3643) .. controls
      (62.1945,60.1803) and (62.3865,60.8043) .. (62.6265,61.8123) .. controls
      (63.1065,63.9723) and (63.0105,66.1803) .. (62.6265,67.7643) .. controls
      (62.5065,68.2683) and (62.1465,69.5403) .. (61.3785,70.4523) .. controls
      (60.3705,71.5563) and (58.9785,72.1323) .. (57.5865,72.1323) .. controls
      (56.2905,72.1323) and (54.8985,71.5563) .. (53.8905,70.4523) .. controls
      (53.1225,69.5403) and (52.7625,68.2683) .. (52.6425,67.7643) .. controls
      (52.2585,66.1803) and (52.1625,63.9723) .. (52.6425,61.8123) .. controls
      (52.8825,60.8043) and (53.0745,60.1803) .. (53.6025,59.3643)(56.1225,63.7563)
      .. controls (55.2345,63.7563) and (54.7785,63.2763) .. (54.5145,62.4843) --
      (54.1545,62.5083) .. controls (54.2505,63.2763) and (54.2985,63.9483) ..
      (54.2985,64.5483) .. controls (54.2985,65.1483) and (54.2505,65.8203) ..
      (54.1545,66.6123) -- (54.5145,66.6363) .. controls (54.7785,65.8203) and
      (55.2345,65.3643) .. (56.1225,65.3643) -- (59.1465,65.3643) .. controls
      (60.0345,65.3643) and (60.4905,65.8203) .. (60.7545,66.6363) --
      (61.1145,66.6123) .. controls (61.0185,65.8203) and (60.9705,65.1483) ..
      (60.9705,64.5483) .. controls (60.9705,63.9483) and (61.0185,63.2763) ..
      (61.1145,62.5083) -- (60.7545,62.4843) .. controls (60.4905,63.2763) and
      (60.0345,63.7563) .. (59.1465,63.7563) -- (56.1225,63.7563);

  \end{scope}
  \begin{scope}[shift={(605.22264,430.57494)},fill=black]% flowRoot3763-8-1-5
    % path5441-2
    \path[fill] (57.6345,73.0683) .. controls (59.6025,73.0683) and
      (61.3785,72.4443) .. (62.7705,71.2923) .. controls (65.0985,69.3723) and
      (65.7225,66.6603) .. (65.7225,64.5963) .. controls (65.7225,62.6283) and
      (65.1225,59.8923) .. (62.7705,57.9963) .. controls (61.3785,56.8683) and
      (59.5545,56.2203) .. (57.6345,56.2203) .. controls (55.7145,56.2203) and
      (53.8905,56.8683) .. (52.4985,57.9963) .. controls (50.1465,59.8923) and
      (49.5465,62.6283) .. (49.5465,64.5963) .. controls (49.5465,66.6603) and
      (50.1705,69.3723) .. (52.4985,71.2923) .. controls (53.8905,72.4443) and
      (55.6665,73.0683) .. (57.6345,73.0683)(53.6025,59.3643) .. controls
      (54.4665,57.9243) and (56.0505,57.1563) .. (57.6345,57.1563) .. controls
      (59.2185,57.1563) and (60.8025,57.9243) .. (61.6665,59.3643) .. controls
      (62.1945,60.1803) and (62.3865,60.8043) .. (62.6265,61.8123) .. controls
      (63.1065,63.9723) and (63.0105,66.1803) .. (62.6265,67.7643) .. controls
      (62.5065,68.2683) and (62.1465,69.5403) .. (61.3785,70.4523) .. controls
      (60.3705,71.5563) and (58.9785,72.1323) .. (57.5865,72.1323) .. controls
      (56.2905,72.1323) and (54.8985,71.5563) .. (53.8905,70.4523) .. controls
      (53.1225,69.5403) and (52.7625,68.2683) .. (52.6425,67.7643) .. controls
      (52.2585,66.1803) and (52.1625,63.9723) .. (52.6425,61.8123) .. controls
      (52.8825,60.8043) and (53.0745,60.1803) .. (53.6025,59.3643)(56.1225,63.7563)
      .. controls (55.2345,63.7563) and (54.7785,63.2763) .. (54.5145,62.4843) --
      (54.1545,62.5083) .. controls (54.2505,63.2763) and (54.2985,63.9483) ..
      (54.2985,64.5483) .. controls (54.2985,65.1483) and (54.2505,65.8203) ..
      (54.1545,66.6123) -- (54.5145,66.6363) .. controls (54.7785,65.8203) and
      (55.2345,65.3643) .. (56.1225,65.3643) -- (59.1465,65.3643) .. controls
      (60.0345,65.3643) and (60.4905,65.8203) .. (60.7545,66.6363) --
      (61.1145,66.6123) .. controls (61.0185,65.8203) and (60.9705,65.1483) ..
      (60.9705,64.5483) .. controls (60.9705,63.9483) and (61.0185,63.2763) ..
      (61.1145,62.5083) -- (60.7545,62.4843) .. controls (60.4905,63.2763) and
      (60.0345,63.7563) .. (59.1465,63.7563) -- (56.1225,63.7563);

  \end{scope}
  \begin{scope}[shift={(860.93693,427.00352)},fill=black]% flowRoot3763-8-1-0
    % path5441-3
    \path[fill] (57.6345,73.0683) .. controls (59.6025,73.0683) and
      (61.3785,72.4443) .. (62.7705,71.2923) .. controls (65.0985,69.3723) and
      (65.7225,66.6603) .. (65.7225,64.5963) .. controls (65.7225,62.6283) and
      (65.1225,59.8923) .. (62.7705,57.9963) .. controls (61.3785,56.8683) and
      (59.5545,56.2203) .. (57.6345,56.2203) .. controls (55.7145,56.2203) and
      (53.8905,56.8683) .. (52.4985,57.9963) .. controls (50.1465,59.8923) and
      (49.5465,62.6283) .. (49.5465,64.5963) .. controls (49.5465,66.6603) and
      (50.1705,69.3723) .. (52.4985,71.2923) .. controls (53.8905,72.4443) and
      (55.6665,73.0683) .. (57.6345,73.0683)(53.6025,59.3643) .. controls
      (54.4665,57.9243) and (56.0505,57.1563) .. (57.6345,57.1563) .. controls
      (59.2185,57.1563) and (60.8025,57.9243) .. (61.6665,59.3643) .. controls
      (62.1945,60.1803) and (62.3865,60.8043) .. (62.6265,61.8123) .. controls
      (63.1065,63.9723) and (63.0105,66.1803) .. (62.6265,67.7643) .. controls
      (62.5065,68.2683) and (62.1465,69.5403) .. (61.3785,70.4523) .. controls
      (60.3705,71.5563) and (58.9785,72.1323) .. (57.5865,72.1323) .. controls
      (56.2905,72.1323) and (54.8985,71.5563) .. (53.8905,70.4523) .. controls
      (53.1225,69.5403) and (52.7625,68.2683) .. (52.6425,67.7643) .. controls
      (52.2585,66.1803) and (52.1625,63.9723) .. (52.6425,61.8123) .. controls
      (52.8825,60.8043) and (53.0745,60.1803) .. (53.6025,59.3643)(56.1225,63.7563)
      .. controls (55.2345,63.7563) and (54.7785,63.2763) .. (54.5145,62.4843) --
      (54.1545,62.5083) .. controls (54.2505,63.2763) and (54.2985,63.9483) ..
      (54.2985,64.5483) .. controls (54.2985,65.1483) and (54.2505,65.8203) ..
      (54.1545,66.6123) -- (54.5145,66.6363) .. controls (54.7785,65.8203) and
      (55.2345,65.3643) .. (56.1225,65.3643) -- (59.1465,65.3643) .. controls
      (60.0345,65.3643) and (60.4905,65.8203) .. (60.7545,66.6363) --
      (61.1145,66.6123) .. controls (61.0185,65.8203) and (60.9705,65.1483) ..
      (60.9705,64.5483) .. controls (60.9705,63.9483) and (61.0185,63.2763) ..
      (61.1145,62.5083) -- (60.7545,62.4843) .. controls (60.4905,63.2763) and
      (60.0345,63.7563) .. (59.1465,63.7563) -- (56.1225,63.7563);

  \end{scope}
  \begin{scope}[shift={(2.0,306.2676)},fill=black]% flowRoot5954
    % path5965
    \path[fill] (37.1797,123.7292) -- (37.1797,128.3699) -- (35.7344,128.3699) --
      (35.7344,116.2917) -- (37.1797,116.2917) -- (37.1797,117.6199) .. controls
      (37.4818,117.0990) and (37.8620,116.7136) .. (38.3203,116.4636) .. controls
      (38.7839,116.2084) and (39.3359,116.0808) .. (39.9766,116.0808) .. controls
      (41.0391,116.0808) and (41.9010,116.5027) .. (42.5625,117.3464) .. controls
      (43.2292,118.1902) and (43.5625,119.2996) .. (43.5625,120.6746) .. controls
      (43.5625,122.0496) and (43.2292,123.1589) .. (42.5625,124.0027) .. controls
      (41.9010,124.8464) and (41.0391,125.2683) .. (39.9766,125.2683) .. controls
      (39.3359,125.2683) and (38.7839,125.1433) .. (38.3203,124.8933) .. controls
      (37.8620,124.6381) and (37.4818,124.2501) ..
      (37.1797,123.7292)(42.0703,120.6746) .. controls (42.0703,119.6173) and
      (41.8516,118.7891) .. (41.4141,118.1902) .. controls (40.9818,117.5860) and
      (40.3854,117.2839) .. (39.6250,117.2839) .. controls (38.8646,117.2839) and
      (38.2656,117.5860) .. (37.8281,118.1902) .. controls (37.3958,118.7891) and
      (37.1797,119.6173) .. (37.1797,120.6746) .. controls (37.1797,121.7318) and
      (37.3958,122.5626) .. (37.8281,123.1667) .. controls (38.2656,123.7657) and
      (38.8646,124.0652) .. (39.6250,124.0652) .. controls (40.3854,124.0652) and
      (40.9818,123.7657) .. (41.4141,123.1667) .. controls (41.8516,122.5626) and
      (42.0703,121.7318) .. (42.0703,120.6746);

    % path5967
    \path[fill] (49.9219,120.6433) .. controls (48.7604,120.6433) and
      (47.9557,120.7761) .. (47.5078,121.0417) .. controls (47.0599,121.3074) and
      (46.8359,121.7605) .. (46.8359,122.4011) .. controls (46.8359,122.9115) and
      (47.0026,123.3178) .. (47.3359,123.6199) .. controls (47.6745,123.9167) and
      (48.1328,124.0652) .. (48.7109,124.0652) .. controls (49.5078,124.0652) and
      (50.1458,123.7839) .. (50.6250,123.2214) .. controls (51.1094,122.6537) and
      (51.3516,121.9011) .. (51.3516,120.9636) -- (51.3516,120.6433) --
      (49.9219,120.6433)(52.7891,120.0495) -- (52.7891,125.0417) --
      (51.3516,125.0417) -- (51.3516,123.7136) .. controls (51.0234,124.2449) and
      (50.6146,124.6381) .. (50.1250,124.8933) .. controls (49.6354,125.1433) and
      (49.0365,125.2683) .. (48.3281,125.2683) .. controls (47.4323,125.2683) and
      (46.7187,125.0183) .. (46.1875,124.5183) .. controls (45.6615,124.0131) and
      (45.3984,123.3386) .. (45.3984,122.4949) .. controls (45.3984,121.5105) and
      (45.7266,120.7683) .. (46.3828,120.2683) .. controls (47.0443,119.7683) and
      (48.0286,119.5183) .. (49.3359,119.5183) -- (51.3516,119.5183) --
      (51.3516,119.3777) .. controls (51.3516,118.7162) and (51.1328,118.2058) ..
      (50.6953,117.8464) .. controls (50.2630,117.4818) and (49.6536,117.2996) ..
      (48.8672,117.2995) .. controls (48.3672,117.2996) and (47.8802,117.3594) ..
      (47.4062,117.4792) .. controls (46.9323,117.5990) and (46.4766,117.7787) ..
      (46.0391,118.0183) -- (46.0391,116.6902) .. controls (46.5651,116.4871) and
      (47.0755,116.3360) .. (47.5703,116.2370) .. controls (48.0651,116.1329) and
      (48.5469,116.0808) .. (49.0156,116.0808) .. controls (50.2812,116.0808) and
      (51.2266,116.4089) .. (51.8516,117.0652) .. controls (52.4766,117.7214) and
      (52.7891,118.7162) .. (52.7891,120.0495);

    % path5969
    \path[fill] (54.9051,120.6600) -- (56.0071,120.6600) -- (57.8911,123.4784) --
      (59.7852,120.6600) -- (60.8872,120.6600) -- (58.4497,124.3011) --
      (61.0497,128.2417) -- (59.9477,128.2417) -- (57.8149,125.0171) --
      (55.6668,128.2417) -- (54.5598,128.2417) -- (57.2664,124.1944) --
      (54.9051,120.6600);

  \end{scope}
  \begin{scope}[shift={(265.17939,302.9939)},fill=black]% flowRoot5954-5
    % path5965-8
    \path[fill] (37.1797,123.7292) -- (37.1797,128.3699) -- (35.7344,128.3699) --
      (35.7344,116.2917) -- (37.1797,116.2917) -- (37.1797,117.6199) .. controls
      (37.4818,117.0990) and (37.8620,116.7136) .. (38.3203,116.4636) .. controls
      (38.7839,116.2084) and (39.3359,116.0808) .. (39.9766,116.0808) .. controls
      (41.0391,116.0808) and (41.9010,116.5027) .. (42.5625,117.3464) .. controls
      (43.2292,118.1902) and (43.5625,119.2996) .. (43.5625,120.6746) .. controls
      (43.5625,122.0496) and (43.2292,123.1589) .. (42.5625,124.0027) .. controls
      (41.9010,124.8464) and (41.0391,125.2683) .. (39.9766,125.2683) .. controls
      (39.3359,125.2683) and (38.7839,125.1433) .. (38.3203,124.8933) .. controls
      (37.8620,124.6381) and (37.4818,124.2501) ..
      (37.1797,123.7292)(42.0703,120.6746) .. controls (42.0703,119.6173) and
      (41.8516,118.7891) .. (41.4141,118.1902) .. controls (40.9818,117.5860) and
      (40.3854,117.2839) .. (39.6250,117.2839) .. controls (38.8646,117.2839) and
      (38.2656,117.5860) .. (37.8281,118.1902) .. controls (37.3958,118.7891) and
      (37.1797,119.6173) .. (37.1797,120.6746) .. controls (37.1797,121.7318) and
      (37.3958,122.5626) .. (37.8281,123.1667) .. controls (38.2656,123.7657) and
      (38.8646,124.0652) .. (39.6250,124.0652) .. controls (40.3854,124.0652) and
      (40.9818,123.7657) .. (41.4141,123.1667) .. controls (41.8516,122.5626) and
      (42.0703,121.7318) .. (42.0703,120.6746);

    % path5967-5
    \path[fill] (49.9219,120.6433) .. controls (48.7604,120.6433) and
      (47.9557,120.7761) .. (47.5078,121.0417) .. controls (47.0599,121.3074) and
      (46.8359,121.7605) .. (46.8359,122.4011) .. controls (46.8359,122.9115) and
      (47.0026,123.3178) .. (47.3359,123.6199) .. controls (47.6745,123.9167) and
      (48.1328,124.0652) .. (48.7109,124.0652) .. controls (49.5078,124.0652) and
      (50.1458,123.7839) .. (50.6250,123.2214) .. controls (51.1094,122.6537) and
      (51.3516,121.9011) .. (51.3516,120.9636) -- (51.3516,120.6433) --
      (49.9219,120.6433)(52.7891,120.0495) -- (52.7891,125.0417) --
      (51.3516,125.0417) -- (51.3516,123.7136) .. controls (51.0234,124.2449) and
      (50.6146,124.6381) .. (50.1250,124.8933) .. controls (49.6354,125.1433) and
      (49.0365,125.2683) .. (48.3281,125.2683) .. controls (47.4323,125.2683) and
      (46.7187,125.0183) .. (46.1875,124.5183) .. controls (45.6615,124.0131) and
      (45.3984,123.3386) .. (45.3984,122.4949) .. controls (45.3984,121.5105) and
      (45.7266,120.7683) .. (46.3828,120.2683) .. controls (47.0443,119.7683) and
      (48.0286,119.5183) .. (49.3359,119.5183) -- (51.3516,119.5183) --
      (51.3516,119.3777) .. controls (51.3516,118.7162) and (51.1328,118.2058) ..
      (50.6953,117.8464) .. controls (50.2630,117.4818) and (49.6536,117.2996) ..
      (48.8672,117.2995) .. controls (48.3672,117.2996) and (47.8802,117.3594) ..
      (47.4062,117.4792) .. controls (46.9323,117.5990) and (46.4766,117.7787) ..
      (46.0391,118.0183) -- (46.0391,116.6902) .. controls (46.5651,116.4871) and
      (47.0755,116.3360) .. (47.5703,116.2370) .. controls (48.0651,116.1329) and
      (48.5469,116.0808) .. (49.0156,116.0808) .. controls (50.2812,116.0808) and
      (51.2266,116.4089) .. (51.8516,117.0652) .. controls (52.4766,117.7214) and
      (52.7891,118.7162) .. (52.7891,120.0495);

    % path5969-6
    \path[fill] (54.9051,120.6600) -- (56.0071,120.6600) -- (57.8911,123.4784) --
      (59.7852,120.6600) -- (60.8872,120.6600) -- (58.4497,124.3011) --
      (61.0497,128.2417) -- (59.9477,128.2417) -- (57.8149,125.0171) --
      (55.6668,128.2417) -- (54.5598,128.2417) -- (57.2664,124.1944) --
      (54.9051,120.6600);

  \end{scope}
  \begin{scope}[shift={(506.60796,304.42247)},fill=black]% flowRoot5954-1
    % path5965-5
    \path[fill] (37.1797,123.7292) -- (37.1797,128.3699) -- (35.7344,128.3699) --
      (35.7344,116.2917) -- (37.1797,116.2917) -- (37.1797,117.6199) .. controls
      (37.4818,117.0990) and (37.8620,116.7136) .. (38.3203,116.4636) .. controls
      (38.7839,116.2084) and (39.3359,116.0808) .. (39.9766,116.0808) .. controls
      (41.0391,116.0808) and (41.9010,116.5027) .. (42.5625,117.3464) .. controls
      (43.2292,118.1902) and (43.5625,119.2996) .. (43.5625,120.6746) .. controls
      (43.5625,122.0496) and (43.2292,123.1589) .. (42.5625,124.0027) .. controls
      (41.9010,124.8464) and (41.0391,125.2683) .. (39.9766,125.2683) .. controls
      (39.3359,125.2683) and (38.7839,125.1433) .. (38.3203,124.8933) .. controls
      (37.8620,124.6381) and (37.4818,124.2501) ..
      (37.1797,123.7292)(42.0703,120.6746) .. controls (42.0703,119.6173) and
      (41.8516,118.7891) .. (41.4141,118.1902) .. controls (40.9818,117.5860) and
      (40.3854,117.2839) .. (39.6250,117.2839) .. controls (38.8646,117.2839) and
      (38.2656,117.5860) .. (37.8281,118.1902) .. controls (37.3958,118.7891) and
      (37.1797,119.6173) .. (37.1797,120.6746) .. controls (37.1797,121.7318) and
      (37.3958,122.5626) .. (37.8281,123.1667) .. controls (38.2656,123.7657) and
      (38.8646,124.0652) .. (39.6250,124.0652) .. controls (40.3854,124.0652) and
      (40.9818,123.7657) .. (41.4141,123.1667) .. controls (41.8516,122.5626) and
      (42.0703,121.7318) .. (42.0703,120.6746);

    % path5967-2
    \path[fill] (49.9219,120.6433) .. controls (48.7604,120.6433) and
      (47.9557,120.7761) .. (47.5078,121.0417) .. controls (47.0599,121.3074) and
      (46.8359,121.7605) .. (46.8359,122.4011) .. controls (46.8359,122.9115) and
      (47.0026,123.3178) .. (47.3359,123.6199) .. controls (47.6745,123.9167) and
      (48.1328,124.0652) .. (48.7109,124.0652) .. controls (49.5078,124.0652) and
      (50.1458,123.7839) .. (50.6250,123.2214) .. controls (51.1094,122.6537) and
      (51.3516,121.9011) .. (51.3516,120.9636) -- (51.3516,120.6433) --
      (49.9219,120.6433)(52.7891,120.0495) -- (52.7891,125.0417) --
      (51.3516,125.0417) -- (51.3516,123.7136) .. controls (51.0234,124.2449) and
      (50.6146,124.6381) .. (50.1250,124.8933) .. controls (49.6354,125.1433) and
      (49.0365,125.2683) .. (48.3281,125.2683) .. controls (47.4323,125.2683) and
      (46.7187,125.0183) .. (46.1875,124.5183) .. controls (45.6615,124.0131) and
      (45.3984,123.3386) .. (45.3984,122.4949) .. controls (45.3984,121.5105) and
      (45.7266,120.7683) .. (46.3828,120.2683) .. controls (47.0443,119.7683) and
      (48.0286,119.5183) .. (49.3359,119.5183) -- (51.3516,119.5183) --
      (51.3516,119.3777) .. controls (51.3516,118.7162) and (51.1328,118.2058) ..
      (50.6953,117.8464) .. controls (50.2630,117.4818) and (49.6536,117.2996) ..
      (48.8672,117.2995) .. controls (48.3672,117.2996) and (47.8802,117.3594) ..
      (47.4062,117.4792) .. controls (46.9323,117.5990) and (46.4766,117.7787) ..
      (46.0391,118.0183) -- (46.0391,116.6902) .. controls (46.5651,116.4871) and
      (47.0755,116.3360) .. (47.5703,116.2370) .. controls (48.0651,116.1329) and
      (48.5469,116.0808) .. (49.0156,116.0808) .. controls (50.2812,116.0808) and
      (51.2266,116.4089) .. (51.8516,117.0652) .. controls (52.4766,117.7214) and
      (52.7891,118.7162) .. (52.7891,120.0495);

    % path5969-1
    \path[fill] (54.9051,120.6600) -- (56.0071,120.6600) -- (57.8911,123.4784) --
      (59.7852,120.6600) -- (60.8872,120.6600) -- (58.4497,124.3011) --
      (61.0497,128.2417) -- (59.9477,128.2417) -- (57.8149,125.0171) --
      (55.6668,128.2417) -- (54.5598,128.2417) -- (57.2664,124.1944) --
      (54.9051,120.6600);

  \end{scope}
  \begin{scope}[shift={(772.32225,301.56533)},fill=black]% flowRoot5954-6
    % path5965-83
    \path[fill] (37.1797,123.7292) -- (37.1797,128.3699) -- (35.7344,128.3699) --
      (35.7344,116.2917) -- (37.1797,116.2917) -- (37.1797,117.6199) .. controls
      (37.4818,117.0990) and (37.8620,116.7136) .. (38.3203,116.4636) .. controls
      (38.7839,116.2084) and (39.3359,116.0808) .. (39.9766,116.0808) .. controls
      (41.0391,116.0808) and (41.9010,116.5027) .. (42.5625,117.3464) .. controls
      (43.2292,118.1902) and (43.5625,119.2996) .. (43.5625,120.6746) .. controls
      (43.5625,122.0496) and (43.2292,123.1589) .. (42.5625,124.0027) .. controls
      (41.9010,124.8464) and (41.0391,125.2683) .. (39.9766,125.2683) .. controls
      (39.3359,125.2683) and (38.7839,125.1433) .. (38.3203,124.8933) .. controls
      (37.8620,124.6381) and (37.4818,124.2501) ..
      (37.1797,123.7292)(42.0703,120.6746) .. controls (42.0703,119.6173) and
      (41.8516,118.7891) .. (41.4141,118.1902) .. controls (40.9818,117.5860) and
      (40.3854,117.2839) .. (39.6250,117.2839) .. controls (38.8646,117.2839) and
      (38.2656,117.5860) .. (37.8281,118.1902) .. controls (37.3958,118.7891) and
      (37.1797,119.6173) .. (37.1797,120.6746) .. controls (37.1797,121.7318) and
      (37.3958,122.5626) .. (37.8281,123.1667) .. controls (38.2656,123.7657) and
      (38.8646,124.0652) .. (39.6250,124.0652) .. controls (40.3854,124.0652) and
      (40.9818,123.7657) .. (41.4141,123.1667) .. controls (41.8516,122.5626) and
      (42.0703,121.7318) .. (42.0703,120.6746);

    % path5967-9
    \path[fill] (49.9219,120.6433) .. controls (48.7604,120.6433) and
      (47.9557,120.7761) .. (47.5078,121.0417) .. controls (47.0599,121.3074) and
      (46.8359,121.7605) .. (46.8359,122.4011) .. controls (46.8359,122.9115) and
      (47.0026,123.3178) .. (47.3359,123.6199) .. controls (47.6745,123.9167) and
      (48.1328,124.0652) .. (48.7109,124.0652) .. controls (49.5078,124.0652) and
      (50.1458,123.7839) .. (50.6250,123.2214) .. controls (51.1094,122.6537) and
      (51.3516,121.9011) .. (51.3516,120.9636) -- (51.3516,120.6433) --
      (49.9219,120.6433)(52.7891,120.0495) -- (52.7891,125.0417) --
      (51.3516,125.0417) -- (51.3516,123.7136) .. controls (51.0234,124.2449) and
      (50.6146,124.6381) .. (50.1250,124.8933) .. controls (49.6354,125.1433) and
      (49.0365,125.2683) .. (48.3281,125.2683) .. controls (47.4323,125.2683) and
      (46.7187,125.0183) .. (46.1875,124.5183) .. controls (45.6615,124.0131) and
      (45.3984,123.3386) .. (45.3984,122.4949) .. controls (45.3984,121.5105) and
      (45.7266,120.7683) .. (46.3828,120.2683) .. controls (47.0443,119.7683) and
      (48.0286,119.5183) .. (49.3359,119.5183) -- (51.3516,119.5183) --
      (51.3516,119.3777) .. controls (51.3516,118.7162) and (51.1328,118.2058) ..
      (50.6953,117.8464) .. controls (50.2630,117.4818) and (49.6536,117.2996) ..
      (48.8672,117.2995) .. controls (48.3672,117.2996) and (47.8802,117.3594) ..
      (47.4062,117.4792) .. controls (46.9323,117.5990) and (46.4766,117.7787) ..
      (46.0391,118.0183) -- (46.0391,116.6902) .. controls (46.5651,116.4871) and
      (47.0755,116.3360) .. (47.5703,116.2370) .. controls (48.0651,116.1329) and
      (48.5469,116.0808) .. (49.0156,116.0808) .. controls (50.2812,116.0808) and
      (51.2266,116.4089) .. (51.8516,117.0652) .. controls (52.4766,117.7214) and
      (52.7891,118.7162) .. (52.7891,120.0495);

    % path5969-2
    \path[fill] (54.9051,120.6600) -- (56.0071,120.6600) -- (57.8911,123.4784) --
      (59.7852,120.6600) -- (60.8872,120.6600) -- (58.4497,124.3011) --
      (61.0497,128.2417) -- (59.9477,128.2417) -- (57.8149,125.0171) --
      (55.6668,128.2417) -- (54.5598,128.2417) -- (57.2664,124.1944) --
      (54.9051,120.6600);

  \end{scope}
\end{scope}

\end{tikzpicture}

\caption{\label{learnex}Variational approximation of parameter ($\Theta$) learning in a Bayesian network with a fully factorized approximation. \textbf{A}: fragment of a Bayesian network. $\pa{\Xv}$ are
the parents of $\Xv$, which may have parents and other children. For learning the parameters $\Theta$, these other children/parents are not relevant. \textbf{B}: corresponding factor graph
fragment. The parameters $\Theta$ appear only in the factor connecting $\Xv$ to its parents. \textbf{C}: fully factorized approximation and \textbf{D} corresponding factor graph. When
computing the variational bound, $\log(P(\Xv|\Theta,\pa{\Xv}))$ has to be averaged over all variables that appear in it, which are the neighbours of the factor node $P(\Xv|\Theta,\pa{\Xv})$ in \textbf{B}.}

\end{figure}

To apply variational message passing, it is necessary to know the factors, which are the conditional probability distributions in case of a Bayesian network. If we want to learn these factors
from data, then it is useful to have a compact parametrization of them, which can be done with exponential family distributions and their conjugate p(oste)riors. Consider the network (fragment)
in fig. \ref{learnex}. Assume we wanted to learn the conditional distribution of $\Xv$ given its parents, and parametrize this distribution by $\Theta$. We lump these parents together in one
supernode. $\Xv$ may be continuous or discrete, but we assume that the parents of $\Xv$, $\pa{\Xv}$ are discrete (in some special cases, continuous models are tractable). 
Also assume that the distribution of $\Xv$ given $\pa{\Xv}$ is from the exponential family, i.e.

\begin{equation}
 p(\Xv|\pa{\Xv},\etav_{\pa{\Xv}}) = h(\xv)g(\etav_{\pa{\Xv}})\exp(\etav_{\pa{\Xv}}^T \uv(\xv))
\end{equation}

i.e. there is one parameter vector $\etav_{\pa{\Xv}}$ for each value of $\pa{\Xv}$, and $\Theta$ is the concatentation of these parameter vectors.
The conjugate prior on each $\etav_{\pa{\Xv}}$ is then

\begin{equation}
 \label{cthetaprior}
 p(\etav_{\pa{\Xv}}|\lambdav_{\pa{\Xv}},\nu_{\pa{\Xv}}) =f(\lambdav_{\pa{\Xv}},\nu_{\pa{\Xv}}) g(\etav_{\pa{\Xv}})^\nu \exp(\nu_{\pa{\Xv}} \etav_{\pa{\Xv}}^T \lambdav_{\pa{\Xv}})
\end{equation}

\noindent
Assume now we had observed $n=1,\ldots,N$ datapoints $\dv_{1:N}$ and computed the corresponding latent variable distributions $Q(X^n)$, $Q(\pa{\Xv}^n)$. If any of the nodes are observed,
replace the corresponding distribution with a distribution concentrated at the observed value. Note that in a fully factorized approximation,
each $Q(\pa{\Xv^n})$ is a actually a product over the parents, $Q(\pa{\Xv^n})= \prod_{\Yv^n \in \pa{\Xv^n}} Q(Y^n)$. Looking at fig. \ref{learnex}, we note that there is exactly one factor connecting
the parameter node to $\Xv$ and $\pa{\Xv}$, and the prior factor for $\Theta$. Using an approximating posterior for $Q(\Theta)$ which has the same form as the prior (eqn. \ref{cthetaprior}), 
the summands in the variational bound which depend on the posterior distribution of $\Theta$ are (where $\yv^n,\yv \in \mbox{range}(\pa{\Xv^n})$):

\begin{eqnarray}
 {\mathcal L}_{\Theta} &=& \sum_{n=1}^{N} \sum_{\xv^n} \sum_{\yv^n} Q(\xv^n) Q(\yv^n) \left\langle \log(P(\xv^n|\Theta,\yv^n)) \right\rangle_{Q(\Theta)} - \beta D(Q(\Theta)||P(\Theta)) \nonumber \\
&=&  \sum_{n=1}^{N} \sum_{\xv^n} \sum_{\yv^n} Q(\xv^n) Q(\yv^n) \left\langle \log(P(\xv^n|\etav_{\yv^n})) \right\rangle_{Q(\etav_{\yv^n}|\lambdavt_{\yv^n},\nut_{\yv^n})} \nonumber \\
&& - \sum_{\yv} \beta D(Q(\etav_\yv|\lambdavt_\yv,\nut_\yv)||P(\etav_{\yv}|\lambdav_\yv,\nu_\yv))
\end{eqnarray}

\noindent
We rewrite the first term as a sum over the possible values of $\xv$ and $\pa{X}$:

\begin{eqnarray}
 {\mathcal L}_{\Theta} &=& \sum_{\xv} \sum_{\yv}  \left\langle \log(P(\xv|\etav_{\yv})) \right\rangle_{Q(\etav_{\yv}|\lambdavt_{\yv},\nut_{\yv})} \sum_{n=1}^{N} Q(\Xv^n=\xv) Q(\pa{X^n}=\yv) \nonumber \\
 & & - \sum_{\yv} \beta D(Q(\etav_\yv|\lambdavt_\yv,\nut_\yv)||P(\etav_{\yv}|\lambdav_\yv,\nu_\yv))
\end{eqnarray}
and define the "responsibilities" (because they measure how much a given setting of the latent variables contributes to explaining a datapoint)
\begin{equation}
 r_{\xv,\yv}=  \sum_{n=1}^{N} Q(\Xv^n=\xv) Q(\pa{X^n}=\yv)
\end{equation}

\noindent
Using the definition of the exponential familiy distribution (eqn. \ref{expofam}), the Kullback-Leibler divergence between conjugate p(oste)riors (eqn. \ref{KLprior}) and denoting
$Q(\etav_{\yv})=Q(\etav_{\yv}|\lambdavt_{\yv},\nut_{\yv})$, we find

\begin{eqnarray}\label{Ltheta}
 {\mathcal L}_{\Theta} &=& \sum_{\xv} \sum_{\yv} \left[ \log(h(\xv)) + \langle \log(g(\etav_\yv)) \rangle_{Q(\etav_{\yv})} + \uv(\xv)^T  \langle \etav_\yv \rangle_{Q(\etav_{\yv)}}  \right] r_{\xv,\yv} \nonumber \\
 && - \sum_{\yv} \left[ \beta \log\left(\frac{f(\lambdavt_\yv,\nut_\yv)}{f(\lambdav_\yv,\nu_\yv)}\right) - \beta(\nut_\yv-\nu_\yv) \frac{\partial \log(f(\lambdavt_\yv,\nut_\yv))}{\partial \nut_\yv} + \beta(\lambdavt_\yv^T - \lambdav_\yv^T) \nu_\yv \langle \etav_\yv \rangle_{Q(\etav_{\yv})} \right] \nonumber \\
\end{eqnarray}

\noindent
The expectations $\langle \log(g(\etav_\yv)) \rangle_{Q(\etav_{\yv})}$ can be expressed using eqn. \ref{gexp}:

\begin{equation}
 \langle \log(g(\etav_\yv)) \rangle_{Q(\etav_{\yv})} = - \frac{\partial \log(f(\lambdavt_\yv,\nut_\yv)}{\partial \nut_\yv} - \lambdavt_\yv^T \langle \etav_\yv \rangle_{Q(\etav_{\yv})}
\end{equation}

Inserting this expression into eqn. \ref{Ltheta}:

\begin{eqnarray}
 {\mathcal L}_{\Theta} &=& \sum_{\xv} \sum_{\yv} \left[ \log(h(\xv))  - \frac{\partial \log(f(\lambdavt_\yv,\nut_\yv)}{\partial \nut_\yv} - \lambdavt_\yv^T \langle \etav_\yv \rangle_{Q(\etav_{\yv})}  + \uv(\xv)^T  \langle \etav_\yv \rangle_{Q(\etav_{\yv)}}  \right] r_{\xv,\yv} \nonumber \\
 && - \sum_{\yv} \left[ \beta \log\left(\frac{f(\lambdavt_\yv,\nut_\yv)}{f(\lambdav_\yv,\nu_\yv)}\right) - \beta(\nut_\yv-\nu_\yv) \frac{\partial \log(f(\lambdavt_\yv,\nut_\yv))}{\partial \nut_\yv} + \beta(\lambdavt_\yv^T - \lambdav_\yv^T) \nu_\yv \langle \etav_\yv \rangle_{Q(\etav_{\yv})} \right] \nonumber \\
\end{eqnarray}



\noindent
Collecting terms, we find:
\begin{eqnarray}
 {\mathcal L}_{\Theta} &=& \sum_{\xv} \sum_{\yv} \log(h(\xv)) r_{\xv,\yv} - \sum_{\yv}  \beta \log\left(\frac{f(\lambdavt_\yv,\nut_\yv)}{f(\lambdav_\yv,\nu_\yv)}\right) \nonumber \\
 && + \sum_{\yv} \frac{\partial \log(f(\lambdavt_\yv,\nut_\yv)}{\partial \nut_\yv} \left[ \beta\left(\nut_\yv - \nu_\yv\right) - \sum_{\xv} r_{\xv,\yv} \right] \nonumber \\
 && -\sum_{\yv} \nu_\yv  \left[\beta \left(\lambdavt_\yv^T - \lambdav_\yv^T \right)+ 
 \lambdavt_\yv^T \frac{\sum_\xv r_{\xv,\yv}}{\nu_\yv}- \frac{\sum_{\xv} r_{\xv,\yv} \uv(\xv)^T}{\nu_\yv}  \right] \langle \etav_\yv \rangle_{Q(\etav_\yv)} \nonumber \\
 \end{eqnarray}


\noindent
With the expressions
\begin{eqnarray}
 \hat{\nu}_\yv &=& \nu_\yv + \frac{\sum_{\xv,\yv} r_{\xv,\yv}}{\beta} = \nu_\yv \left( 1+ \frac{\sum_\xv r_{\xv,\yv}}{\nu_\yv\beta} \right) \nonumber \\
 \hat{\lambdav}_\yv &=& \frac{\nu_\yv \lambdav_\yv + \frac{\sum_\xv r_{\xv,\yv}}{\beta} \uv(\xv)}{\nu_\yv + \frac{\sum_\xv r_{\xv,\yv}}{\beta}} = \frac{\nu_\yv}{\hat{\nu}_\yv} \left( \lambdav_\yv + \frac{\frac{\sum_\xv r_{\xv,\yv}}{\beta} \uv(\xv)}{\nu_\yv}\right)
\end{eqnarray}
and noting that

\begin{equation}
 -\beta \hat{\nu}_\yv = -\beta \nu_\yv - \sum_{\xv,\yv} r_{\xv,\yv}
\end{equation}

\begin{eqnarray}
 \nu_\yv \left[\beta \lambdavt_\yv^T + \lambdavt_\yv^T \frac{\sum_\xv r_{\xv,\yv}}{\nu_\yv}\right] &=&  \beta \nu_\yv  \left[ \lambdavt_\yv^T + \lambdavt_\yv^T \frac{\sum_\xv r_{\xv,\yv}}{\nu_\yv \beta}\right]\nonumber \\
 &=& \beta \nu_\yv \lambdavt_\yv^T  \left[ 1+ \frac{\sum_\xv r_{\xv,\yv}}{\nu_\yv \beta}\right] = \beta \hat{\nu}_\yv \lambdavt_\yv^T 
 \end{eqnarray}
 
 \begin{eqnarray}
  \nu_\yv \left[ -\beta \lambdav_\yv^T - \frac{\sum_{\xv} r_{\xv,\yv} \uv(\xv)^T}{\nu_\yv} \right] & = & - \nu_\yv \beta \left[ \lambdav_\yv^T + \frac{\sum_{\xv} r_{\xv,\yv} \uv(\xv)^T}{\nu_\yv \beta} \right] \nonumber \\
 &=& -\beta \hat{\nu}_\yv \hat{\lambdav}_\yv^T
\end{eqnarray}




we insert a zero by adding and subtracting the term $\sum_{\yv} \beta \log\left(\frac{f(\lambdavt_\yv,\nut_\yv)}{f(\hat{\lambdav}_\yv,\hat{\nu}_\yv)}\right)$ and write (using eqn. \ref{KLprior})
\begin{eqnarray}
 {\mathcal L}_{\Theta} &=& \sum_{\xv} \sum_{\yv} \log(h(\xv)) r_{\xv,\yv} -  \sum_{\yv} \beta \log\left(\frac{f(\lambdavt_\yv,\nut_\yv)}{f(\lambdav_\yv,\nu_\yv)}\right) \nonumber \\
 & & +  \sum_{\yv} \beta \log\left(\frac{f(\lambdavt_\yv,\nut_\yv)}{f(\hat{\lambdav}_\yv,\hat{\nu}_\yv)}\right) - \sum_{\yv} \beta \log\left(\frac{f(\lambdavt_\yv,\nut_\yv)}{f(\hat{\lambdav}_\yv,\hat{\nu}_\yv)}\right) \nonumber \\
& & +  \sum_{\yv} \frac{\partial \log(f(\lambdavt_\yv,\nut_\yv)}{\partial \nut_\yv} \left. \beta(\nut_\yv - \hat{\nu}_\yv \right) \nonumber \\
& & -  \sum_{\yv} \beta\hat{\nu}_\yv (\lambdavt_\yv^T - \hat{\lambdav}_\yv^T) \langle \etav_\yv \rangle_{Q(\etav_\yv)} \\
&=& \sum_{\xv} \sum_{\yv} \log(h(\xv)) r_{\xv,\yv} - \sum_{\yv} \beta  \log\left(\frac{f(\hat{\lambdav}_\yv,\hat{\nu}_\yv)}{f(\lambdav_\yv,\nu_\yv)}\right) \nonumber \\
& & - \sum_{\yv}  \beta D(Q(\etav_\yv|\lambdavt_\yv,\nut_\yv)||Q(\etav_\yv|\hat{\lambdav}_\yv,\hat{\nu}_\yv))
\end{eqnarray}

\noindent
The first part of this expression is constant w.r.t. to the $\lambdavt_\yv,\, \nut_\yv$, and the second part is a sum of KL-divergences. To maximize ${\mathcal L}_{\Theta}$, we therefore
choose the posterior parameters so that each KL-divergence is zero, i.e. $\lambdavt_\yv=\hat{\lambdav}_\yv$ and $\nut_\yv=\hat{\nu}_\yv$:

\vspace{0.3cm}
\begin{mdframed}[linewidth=1pt]
\begin{center}
 {\large \textit{variational posterior parameters}}
\end{center}
\begin{eqnarray}
  \label{rdef}
  r_{\xv,\yv} &:=&  \sum_{n=1}^{N} Q(\Xv^n=\xv) Q(\pa{X^n}=\yv) \\
  \label{nuup_var}
  \nut_\yv &:=& \nu_\yv + \frac{\sum_{\xv} r_{\xv,\yv}}{\beta} \\
  \label{lamup_var}
  \lambdavt_\yv &:=& \frac{\nu_\yv \lambdav_\yv + \frac{\sum_\xv r_{\xv,\yv}}{\beta} \uv(\xv)}{\nu_\yv + \frac{\sum_\xv r_{\xv,\yv}}{\beta}}
\end{eqnarray}
\end{mdframed}

\vspace{0.2cm}
Comparing these learning rules to the conjugate update rules (eqns. \ref{nuup},\ref{lamup}), we see that in the variational framework, a datapoint can be "shared" between different values
of the latent variables in the model. This sharing comes about because the responsibilities are (approximate) probability, rather than deterministic 0s or 1s. Otherwise the rules are identical.
Note in particular that if $r_{\xv,\yv} \in \{0,1\}$, i.e. the variables are known with certainty, then the variational rules reduce to the exact update rules. Also, the $\beta$-variational update rules can be obtained by dividing all responsibilities by $\beta$. Thus, if $\beta>1$, the prior becomes 'stiffer' and tends to ignore the data, whereas for $\beta \to 0$, we get maximum likelihood updates.


\section{Frequently used special cases}\label{speccase}

This section contains frequently used conjugate pairs and relevant quanitities computed thereof.

\subsection{Bernoulli-Beta}

For a discrete random variable $x \in \{0;1\}$, where 1 is alternatively called "success" (e.g. when betting on coin tosses), is given by

\begin{equation}
 \label{bern}
 P(x|q)=q^x (1-q)^{(1-x)}.
\end{equation}
Its canonical conjugate prior is a Beta distribution in $q$ with density

\begin{equation}
 \label{betadens}
 p(q|\alpha,\beta) = \frac{1}{B(\alpha,\beta)} q^{\alpha-1} (1-q)^{\beta-1}.
\end{equation}
To transform these expressions into the exponential family normal form (eqns. \ref{expofam} and \ref{cprior}), introduce the \textit{logit} 
\begin{equation}
 \eta=\log\left( \frac{q}{1-q}\right)
\end{equation}
whence $q=\frac{1}{1+\exp(-\eta)}$, $1-q=\frac{1}{1+\exp(\eta)}$ and $\frac{d q}{d \eta}= -\frac{\exp(-\eta)}{(1+\exp(-\eta))^2} = -q (1-q)$. Substitute $\eta$ in eqn. (\ref{bern}):

\begin{eqnarray}
 P(x|\eta)&=&\exp\left( x \log(q) + (1-x) \log(1-q) \right) \nonumber \\
 &=& \exp\left( x \log\left( \frac{q}{1-q} \right)  + \log(1-q)\right) \nonumber \\
 &=& (1-q) \exp( \eta x ) = \frac{1}{1+\exp(\eta) } \exp( \eta x )
\end{eqnarray}

Hence, $h(x)=1$, $g(\eta) = \frac{1}{1+\exp(\eta) }$ (cf. eqn. (\ref{expofam})) and

\begin{equation}
 P(x|\eta) = h(x) g(\eta) \exp(\eta x).
\end{equation}
To transform the Beta density into exponential family normal form, note that densities transform like $p(\eta) = p(q(\eta)) \left| \frac{dq}{d\eta}\right|$:
\begin{eqnarray}
 p(\eta|\alpha,\beta) &=&  \frac{1}{B(\alpha,\beta)} q^{\alpha-1} (1-q)^{\beta-1} \left| \frac{dq}{d\eta}\right| \nonumber \\
 &=& \frac{1}{B(\alpha,\beta)} q^{-1}(1-q)^{-1} \exp\left( \alpha \log(q) +  \beta \log(1-q) \right) \, q(1-q) \nonumber \\
 &=& \frac{1}{B(\alpha,\beta)} \exp\left( \alpha \log\left(\frac{q}{1-q}\right) +  (\alpha+\beta) \log(1-q) \right)  \nonumber \\
 &=& \frac{1}{B(\alpha,\beta)} \left[\frac{1}{1+\exp(\eta)}\right]^{\alpha+\beta} \exp\left( \alpha \eta \right)
\end{eqnarray}

Letting $\nu:=\alpha+\beta$, $\lambda := \frac{\alpha}{\nu}$, $f(\lambda,\nu)=\frac{1}{B(\nu \lambda, \nu (1-\lambda))}$, $m(\eta)=1$ we find
\begin{eqnarray}
 p(\eta|\lambda,\nu) &=& \frac{1}{B(\nu \lambda, \nu (1-\lambda))} \left[\frac{1}{1+\exp(\eta)}\right]^{\nu} \exp(\nu \lambda \eta) \nonumber \\
 &=& f(\lambda,\nu) \, m(\eta)\,  g(\eta)^\nu \exp(\nu \lambda \eta).
\end{eqnarray}

\begin{table}[h!]
\centering
\begin{tabular}{c|c}
 \multicolumn{2}{c}{\textbf{Bernoulli distribution}} \\ \hline \hline
 standard form & $q^x(1-q)^{1-x}$ \\ \hline
 constraints $x$ & $x \in \{0,1\}$ \\ \hline
 constraints $q$ & $q \in [0,1]$ \\ \hline
 $u(x)$ & $x$ \\ \hline
 $\eta$ & $\log\left( \frac{q}{1-q}\right)$ \\ \hline
 $q$ & $\frac{1}{1+\exp(-\eta)}$ \\ \hline
 constraints $\eta$ & $\eta \in \mathbb{R}$ \\ \hline
 $g(\eta)$ & $\frac{1}{1+\exp(\eta) }=1-q$ \\ \hline
 $h(x)$ & 1 \\ \hline
 $\langle u(x) \rangle$ & $\frac{1}{1+\exp(-\eta)}=q$ \\ \hline
 $\mbox{Var}(u(x))$ & $\frac{\exp(-\eta)}{(1+\exp(-\eta))^2}=q(1-q)$
\end{tabular}
\vspace*{0.2cm}

\begin{tabular}{c|c}
 \multicolumn{2}{c}{\textbf{Beta distribution}} \\ \hline \hline
 standard form & $\frac{q^{\alpha-1}(1-q)^{\beta-1}}{B(\alpha,\beta)}$ \\ \hline
 constraints $\alpha,\beta$ & $\alpha,\beta \in \mathbb{R}^+$ \\ \hline
 $\lambda$ & $\frac{\alpha}{\alpha+\beta}$ \\ \hline
 $\nu$ & $\alpha+\beta$ \\ \hline
 $\alpha$ & $\nu \lambda$ \\ \hline
 $\beta$ & $\nu(1-\lambda)$ \\ \hline
 constraints $\nu$ & $\nu \in \mathbb{R}^+$ \\ \hline
 constraints $\lambda$ & $\lambda \in [0,1]$ \\ \hline
 $f(\lambda,\nu)$ & $\frac{1}{B\left(\nu\lambda,\nu(1-\lambda)\right)}$ \\ \hline
 $m(\eta)$ & 1 \\ \hline
 $\langle \eta \rangle$ & $\psi(\nu\lambda)-\psi(\nu(1-\lambda))$ \\ \hline
 $\mbox{Var}(\eta)$ & $\frac{\psi(\nu\lambda)-\psi(\nu(1-\lambda))}{\nu^2} + \psi^\prime(\nu\lambda)+\psi^\prime(\nu(1-\lambda))$ \\ \hline
 $\frac{\partial \log(f(\lambda,\nu))}{\partial \nu}$ & $ \psi(\nu) - \lambda \psi(\nu\lambda) - (1-\lambda)\psi(\nu(1-\lambda)) $ \\ \hline
 $\frac{\partial^2 \log(f(\lambda,\nu))}{\partial \nu^2}$ & $ \psi^\prime(\nu) - \lambda^2 \psi^\prime(\nu\lambda) - (1-\lambda)^2\psi^\prime(\nu(1-\lambda))$ \\ \hline
 $\langle g(\eta) \rangle$ & $1-\lambda = \frac{\beta}{\alpha+\beta} = 1 - \langle q \rangle$ \\ \hline
 $\mbox{Var}(g(\eta))$ & $\frac{(1-\lambda)\lambda}{\nu+1} = \frac{\frac{\beta}{\alpha+\beta}\frac{\alpha}{\alpha+\beta}}{\alpha+\beta+1}$ \\ \hline
 $p(x|\lambda,\nu)$ & $\lambda x + (1-\lambda) (1-x) = x \langle q \rangle + (1-x) (1-\langle q\rangle)$ \\ \hline
 $\langle u(x) \rangle_{p(x|\lambda,\nu)}$ & $\lambda$
\end{tabular}
\caption{Bernoulli distribution and conjugate Beta prior}
\end{table}


\subsection{Multinomial-Dirichlet}

The multinomial distribution is the generalization of the Bernoulli distribution to $K$ possible outcomes. It is convenient to represent multinomial random variates \xv by vectors with $K$ components,
such that $x_k \in \{0;1\}$ and $\sum_{k=1}^K x_k = 1$, whence $x_K=1-\sum_{k=1}^{K-1} x_k$. This is called \textit {1-of-K} representation, because exactly one component of \xv is 1 and all others are 0.
Let $\qv=(q_1,\ldots,q_K)$ be the probabilities of the $K$ possible \xv, such that $q_K=1-\sum_{k=1}^{K-1} q_k$. Then, the multinomial distribution can be written as
\begin{equation}
 P(\xv|\qv) = \prod_{k=1}^{K-1} q_k^{x_k} \left( 1- \sum_{k=1}^{K-1} q_k \right)^{x_K}
\end{equation}
This expression can be transformed into exponential family form via
\begin{eqnarray}
 P(\xv|\qv) &=& \exp\left( \sum_{k=1}^{K-1} x_k \log(q_k) + \left(1-\sum_{k=1}^{K-1} x_k \right) \log \left(1-\sum_{k=1}^{K-1} q_k \right) \right) \nonumber \\
 &=& \left(1-\sum_{k=1}^{K-1} q_k \right) \exp\left( \sum_{k=1}^{K-1} x_k \log\left( \frac{q_k}{1-\sum_{i=1}^{K-1} q_i} \right)\right)
\end{eqnarray}
and by introducing the generalized logit $\eta_k=\log\left( \frac{q_k}{1-\sum_{i=1}^{K-1} q_i} \right)$ we find that
\begin{eqnarray}
 \forall k=1,\ldots,K-1 &:& q_k=\frac{\exp(\eta_k)}{1+\sum_{i=1}^{K-1} \exp(\eta_i)} \\
 && q_K=\frac{1}{1+\sum_{i=1}^{K-1} \exp(\eta_i)}
\end{eqnarray}
(alternatively, we could fix $\eta_K=0$, and let $\qv=\mbox{softmax}(\etav)$). Hence, $h(\xv)=1$ (after $x_k \in \{0;1\}$ has been enforced) and 
$$g(\eta) = \left(1-\sum_{k=1}^{K-1} q_k \right)  = q_K = \frac{1}{1+\sum_{i=1}^{K-1} \exp(\eta_i)}$$.

The standard conjugate prior on the multinomial is the Dirichlet distribution. Let $\alphav=(\alpha_1,\ldots,\alpha_K)$ with $\alpha_k \geq 0$ and 
\begin{equation}
 M=\frac{\prod_{k=1}^K \Gamma(\alpha_k)}{\Gamma(\sum_{k=1}^{K} \alpha_k)}
\end{equation}
then the density of the Dirichlet distribution is
\begin{eqnarray}
 p(\qv|\alphav) &=& M \prod_{k=1}^{K-1} q_k^{\alpha_k-1} \left(1-\sum_{k=1}^{K-1} q_k\right)^{\alpha_K-1} \\
 &=& M \exp\left( \sum_{k=1}^{K-1} (\alpha_k-1)\log(q_k) + (\alpha_K-1) \log\left( 1-\sum_{k=1}^{K-1} q_k \right) \right) \nonumber \\
 &=& M \prod_{k=1}^{K-1} q_k^{-1} (1-\sum_{k=1}^{K-1} q_k)^{-1} \exp\left( \sum_{k=1}^{K-1} \alpha_k \log\left( \frac{q_k}{q_K}\right)  + \sum_{k=1}^{K} \alpha_K \log(q_K) \right) \nonumber \\
 &=& M\; q_K^{\sum_{k=1}^K \alpha_K} \prod_{k=1}^{K-1} q_k^{-1} (1-\sum_{k=1}^{K-1} q_k)^{-1} \exp\left( \alphav^T \etav \right) 
\end{eqnarray}
To substitute $\qv$ by $\etav$, we need to compute the determinant of the Jacobian $\frac{d\qv}{d\etav}$. Let $Z=1+\sum_{k=1}^{K-1} \exp(\eta_k)$, i.e. $q_k=\frac{\exp(\eta_k)}{Z}$. Then
\begin{eqnarray}
 \left| \frac{d\qv}{d\etav} \right| &=& \left|\left(\begin{array}{ccc}  \frac{\exp(\eta_1)}{Z} - \frac{\exp(2\eta_1)}{Z^2} & \ldots & -\frac{\exp(\eta_1)\exp(\eta_K)}{Z^2} \\
 & \ddots & \\
 -\frac{\exp(\eta_K)\exp(\eta_1)}{Z^2} & \ldots & \frac{\exp(\eta_K)}{Z} - \frac{\exp(2\eta_K)}{Z^2} \end{array}\right)\right| \nonumber \\
 &=& \left|\left(\begin{array}{ccc}q_1 & \ldots & 0 \\ 0 & \ddots & 0 \\ 0 & \ldots & q_K\end{array}\right) - \left(\begin{array}{c}q_1\\ \vdots \\ q_K  \end{array} \right) \left( q_1 \ldots q_K \right) \right| \nonumber \\
 &=& \left|\left(\begin{array}{ccc}q_1 & \ldots & 0 \\ 0 & \ddots & 0 \\ 0 & \ldots & q_K\end{array}\right) \left[ \mathbf{1}_{K\times K} - \left( \begin{array}{c} 1 \\ \vdots \\ 1 \end{array} \right) \left( q_1 \ldots q_K \right) \right]\right| \nonumber \\
 &=& \prod_{k=1}^{K-1} q_k \left( 1- \sum_{k=1}^{K-1} q_k \right)
\end{eqnarray}
And thus
\begin{eqnarray}
 p(\etav|\alphav) &=& p(\qv(\etav)|\alphav) \left| \frac{d\qv}{d\etav} \right| \nonumber \\
 &=& M \left( \frac{1}{1+\sum_{i=1}^{K-1} \exp(\eta_i)} \right)^{\sum_{k=1}^K \alpha_K} \exp\left( \alphav^T \etav \right)
\end{eqnarray}
With $\nu=\sum_{k=1}^K \alpha_k$, $\lambda_k=\frac{\alpha_k}{\nu}$, $f(\lambdav,\nu)=M$, $m(\etav)=1$ and $g(\etav)$ as above, we find
\begin{equation}
 p(\etav|\lambdav,\nu) = f(\lambdav,\nu) m(\etav) g(\etav)^\nu \exp( \nu \etav \lambdav)
\end{equation}

\begin{table}[h!]
\centering
\begin{tabular}{c|c}
 \multicolumn{2}{c}{\textbf{multinomial distribution}} \\ \hline \hline
 standard form & $\prod_{k=1}^{K-1} q_k^{x_k} \left( 1- \sum_{k=1}^{K-1} q_k \right)^{x_K}$ \\ \hline
 constraints $\xv$ & $x_k \in \{0,1\}$, $\sum_{k=1}^K x_k=1$ \\ \hline
 constraints $\qv$ & $q_k \in [0,1]$, $\sum_{k=1}^{K-1} q_k \leq 1$ \\ \hline
 $\uv(\xv)$ & \xv \\ \hline
 $\etav$ & $\log\left( \frac{\qv}{1-\sum_{i=1}^{K-1} q_i} \right)$ \\ \hline
 $\qv$ & $\frac{\exp(\etav)}{1+\sum_{i=1}^{K-1} \exp(\eta_i)}$ \\ \hline
 constraints $\etav$ & $\eta_k \in \mathbb{R}$ \\ \hline
 $g(\eta)$ & $\frac{1}{1+\sum_{i=1}^{K-1} \exp(\eta_i)}=1-\sum_{i=1}^{K-1} q_i = q_K$ \\ \hline
 $h(x)$ & 1 \\ \hline
 $\langle \uv(\xv) \rangle$ & $\frac{\exp(\etav)}{1+\sum_{i=1}^{K-1} \exp(\eta_i)}=\qv$ \\ \hline
 $\mbox{Cov}(\uv(\xv))$ & $C_{kl}=\frac{\delta_{kl} \exp(\eta_k)}{1+\sum_{i=1}^{K-1} \exp(\eta_i)} - \frac{\exp(\eta_k+\eta_l)}{\left(1+\sum_{i=1}^{K-1} \exp(\eta_i)\right)^2}=\delta_{kl} q_l - q_l q_k$
\end{tabular}
\vspace*{0.2cm}

\begin{tabular}{c|c}
 \multicolumn{2}{c}{\textbf{Dirichlet distribution}} \\ \hline \hline
 standard form & $\frac{\Gamma(\sum_{k=1}^K \alpha_k)}{\prod_{k=1}^K \Gamma(\alpha_k)} \prod_{k=1}^{K-1} q_k^{\alpha_k-1} \left(1-\sum_{k=1}^{K-1} q_k\right)^{\alpha_K-1}$ \\ \hline
 constraints $\alphav$ & $\alpha_k \in \mathbb{R}^+$ \\ \hline
 $\lambda$ & $\forall k=1,\ldots,K-1: \lambda_k=\frac{\alpha_k}{\sum_{i=1}^K \alpha_i}$ \\ \hline
 $\nu$ & $\sum_{i=1}^K \alpha_i$  \\ \hline
 $\alpha$ & $\nu \lambdav$ \\ \hline
 constraints $\nu$ & $\nu \in \mathbb{R}^+$ \\ \hline
 constraints $\lambdav$ & $\lambda_i \in [0,1]$, $\sum_{i=1}^{K-1} \lambda_i \leq 1$ \\ \hline
 $f(\lambdav,\nu)$ & $\frac{\Gamma(\nu)}{\Gamma\left(\nu(1-\sum_{k=1}^{K-1} \lambda_k)\right)\prod_{i=1}^{K-1} \Gamma\left(\nu \lambda_i\right)}$ \\ \hline
 $m(\etav)$ & 1 \\ \hline
 $\langle \etav \rangle$ & $\psi(\nu\lambdav)-\psi\left(\nu(1-\sum_{i=1}^{K-1} \lambda_k)\right)$ \\ \hline
 $\mbox{Cov}(\eta_i,\eta_j)$ & $\delta_{i,j}\psi^\prime(\nu\lambda_i) + \psi^\prime\left(\nu(1-\sum_{i=1}^{K-1} \lambda_k) \right)$ \\ \hline
 $\frac{\partial \log(f(\lambdav,\nu))}{\partial \nu}$ & $\psi(\nu) - (1-\sum_{i=1}^{K-1} \lambda_i) \psi\left( \nu (1-\sum_{i=1}^{K-1} \lambda_i) \right) - \sum_{i=1}^{K-1} \lambda_i \psi(\nu \lambda_i)$ \\ \hline
 $\frac{\partial^2 \log(f(\lambdav,\nu))}{\partial \nu^2}$ & $\psi^\prime(\nu) - (1-\sum_{i=1}^{K-1} \lambda_i)^2 \psi^\prime\left( \nu (1-\sum_{i=1}^{K-1} \lambda_i) \right) - \sum_{i=1}^{K-1} \lambda_i^2 \psi^\prime(\nu \lambda_i)$ \\ \hline
 $\langle g(\etav) \rangle$ & $1-\sum_{i=1}^{K-1} \lambda_i = \frac{\alpha_K}{\sum_{i=1}^K \alpha_i} = \langle q_K \rangle$ \\ \hline
 $\mbox{Var}(g(\etav))$ & $\frac{(1-\sum_{i=1}^{K-1} \lambda_i)\sum_{i=1}^{K-1} \lambda_i}{\nu+1} = \frac{\langle q_K \rangle \langle 1-q_K \rangle}{\nu+1}$ \\ \hline
 $p(\xv|\lambdav,\nu)$ & $ \sum_{i=1}^{K-1} x_i \lambda_i + x_K (1-\sum_{i=1}^{K-1} \lambda_i) = \sum_{i=1}^K x_i \langle q_i \rangle$ \\ \hline
 $\langle \uv(\xv) \rangle_{p(\xv|\lambdav,\nu)}$ & $\lambdav$
\end{tabular}
\caption{Multiomial distribution and conjugate Dirichlet prior}
\end{table}

\clearpage

\subsection{Multinomial-StickBreaking}

The stick-breaking construction is another way of parameterizing multinomial distributions. It has attracted a lot of attention in Machine Learning around 2005, because it is a convenient way of 
representing \textit{infinite} multinomials with a Dirichlet-process prior. To re-parameterize the distribution of a $K$ component multinomial variable \xv (in 1-of-K representation)  with probabilities \qv, introduce the variables $\vv=(v_1,\ldots,v_{K-1})$
such that
\begin{eqnarray}
 q_1 &=& v_1 \\
 q_2 &=& (1-v_1) v_2 \\
 q_3 &=& (1-v_1) (1-v_2) v_3  \\
     &\vdots& \nonumber \\
 q_{K-1} &=& (1-v_1) (1-v_2) \ldots v_{K-1}  \\
 q_K &=& (1-v_1) (1-v_2) \ldots (1-v_{K-1}).
\end{eqnarray}
The distribution of \xv can then be written as
\begin{equation}
 \label{stickBreak}
 p(\xv|\vv) = \prod_{i=1}^{K-1} v_i^{x_i} (1-v_i)^{\sum_{j=i+1}^K x_j}
\end{equation}
We now introduce the sufficient statistics $\uv(\xv)=(u_0,\ldots,u_{K-1})$ 
\begin{equation}
 u_k=\sum_{i=k+1}^K x_i
\end{equation}
hence $u_0=1$. $u_k=1$ implies that $x_i=1$ with $i>k$. Eqn. \ref{stickBreak} becomes
\begin{equation}
 p(\xv|\vv) = \prod_{i=1}^{K-1} v_i^{u_{i-1}-u_i} (1-v_i)^{u_i} = \exp\left( \sum_{i=1}^{K-1} (u_{i-1}-u_i) \log(v_i) + u_i \log(1-v_i)\right)
\end{equation}
To transform this expression into exponential family normal form, rewrite the exponent on the r.h.s. as
\begin{equation}
 \log(v_1) + \sum_{i=1}^{K-2} u_i \log\left( \frac{1-v_i}{v_i} v_{i+1} \right) + u_{K-1} \log\left( \frac{1-v_{K-1}}{v_{K-1}} \right)
\end{equation}
and introduce the natural parameters $\etav=(\eta_1,\ldots,\eta_{K-1})$:
\begin{eqnarray}
 \eta_{K-1}& = &\log\left( \frac{1-v_{K-1}}{v_{K-1}}\right)  \\
 \forall k=K-2,\ldots,1: \eta_{k}&=&\log\left( \frac{1-v_{k}}{v_{k}} v_{k+1}\right)
\end{eqnarray}

Solving for \vv yields
\begin{eqnarray}
 \label{vbyeta1}
 v_{K-1}&=&\frac{1}{1+\exp(\eta_{K-1})} \\
 v_{K-2} &=& \frac{1}{1+\exp(\eta_{K-2})(1+\exp(\eta_{K-1}))} \\
 &\vdots& \nonumber \\
 \label{vbyeta2}
 v_1 &=& \frac{1}{1+\exp(\eta_1)(1+\exp(\eta_2)(1+\ldots(1+exp(\eta_{K-1}))\ldots)}
\end{eqnarray}
Thus
\begin{equation}
 P(\xv|\etav) = v_1(\etav) \exp\left( \etav^T \uv(\xv)\right)
\end{equation}
which is in exponential family normal form, with $h(\xv)=1$ and $g(\etav)=v_1(\etav)$.

The conjugate p(oste)rior on \vv is given by a product of Beta distributions. Let $\alphav,\betav$ be the parameters of these Beta distributions, then
\begin{equation}
 \label{betaprod}
 p(\vv|\alphav,\betav) = \prod_{i=1}^{K-1} \frac{1}{B(\alpha_i,\beta_i)} v_i^{\alpha_i-1} (1-v_i)^{\beta_i-1}
\end{equation}

To derive the corresponding density in \etav we need the determinant of the Jacobian $\left|\frac{d\vv}{d\etav}\right|$. Note that as a consequence of eqns. \ref{vbyeta1}-\ref{vbyeta2}, the Jacobian is
an upper triangular matrix, hence the determinant is given by the product of the diagonal entries:
\begin{eqnarray}
 \frac{dv_i}{d\eta_i}&=& - \frac{\exp(\eta_i)(1+\exp(\eta_{i+1})...)}{\left(1+\exp(\eta_i)(1+\exp(\eta_{i+1})(....)) \right)^2}= -v_i(1-v_i) \\
 \Rightarrow \left|\frac{d\vv}{d\etav}\right| &=& \prod_{i=1}^{K-1} v_i (1-v_i).
\end{eqnarray}
Now we can rewrite eqn. \ref{betaprod} in exponential form
\begin{equation}
 p(\vv|\alphav,\betav)= \prod_{i=1}^{K-1} \frac{v_i^{-1}(1-v_i)^{-1}}{B(\alpha_i,\beta_i)} \exp\left( \sum_{k=1}^{K-1} \alpha_i \log(v_i) + \beta_i \log(1-v_i) \right)
\end{equation}
and rearrange the exponent as:
\begin{eqnarray}
 & &\sum_{k=1}^{K-1} \alpha_i \log(v_i) + \beta_i \log(1-v_i)  \nonumber \\
 &=& \sum_{k=1}^{K-1} (\alpha_i+\beta_i) \log(v_i) + \beta_i \log\left( \frac{1-v_i}{v_i} \right) \\
 &=& \beta_{K-1} \log\left( \frac{1-v_{K-1}}{v_{K-1}} \right) + \sum_{i=1}^{K-2} \beta_i \log\left( \frac{1-v_i}{v_i}v_{i+1} \right) \nonumber \\
 & & + (\alpha_1+\beta_1) \log(v_1) + \sum_{i=2}^{K-1} \underbrace{(\alpha_{i}+\beta_{i}-\beta_{i-1})}_{=:c_i} \log(v_i) \\
 &=& \beta_{K-1} \eta_{K-1} + \sum_{i=1}^{K-2} \beta_i \eta_i + (\alpha_1+\beta_1) \log(g(\etav)) + \sum_{i=2}^{K-1} c_i \log(v_i)
\end{eqnarray}
Hence, letting $N(\alphav,\betav)=\prod_{i=1}^{K-1} B(\alpha_i,\beta_i)^{-1}$ we obtain for the density of \etav:
\begin{eqnarray}
 p(\etav|\alphav,\betav) &=& p(\vv(\etav)|\alpha,\beta) \left| \frac{d\vv}{d\etav} \right| \\
 &=& N(\alphav,\betav) \prod_{i=2}^{K-1} v_i(\etav)^{c_i} g(\etav)^{\alpha_1+\beta_1} \exp\left( \etav^T \betav \right)
\end{eqnarray}
which is almost in exponential family normal form, except for:
\begin{itemize}
 \item $\prod_{i=2}^{K-1} v_i(\etav)^{c_i} $ should be $m(\etav)$, i.e. it must not depend on the data, hence the $c_i$ need to be constant.
 Since \alphav and \betav will be updated on observation, this requires that $\forall i=2,\ldots,K-1:\; \alpha_i=c_i-\beta_i+\beta_{i-1}$. This implies that
 \alphav is determined by \betav up to the $c_i$, except for $\alpha_1$.
 \item $\nu:=\alpha_1+\beta_1$ and $\lambda_i:=\frac{\beta_i}{\nu}$, hence $\alpha_1=\nu(1-\lambda_1)$.
 \item $f(\lambdav,\nu)=N(\alphav(\lambdav,\nu),\nu\lambdav)$
\end{itemize}

In the standard parametrization of the prior on the stick-breaking construction (eqn. \ref{betaprod}), the $\alpha_i$ are the pseudocounts on the number of instances where $x_i=1$. To see that our definition
\begin{equation}
 \alpha_i:=c_i-\beta_i+\beta_{i-1}
\end{equation}
has the same meaning, assume that we had observed $N$ datapoints $\xv^{1:N}$ and corresponding sufficient statistics $\uv^{1:N}$. Using the exponential family update rules (eqns. \ref{nuup} and \ref{lamup}),
we find for the posterior parameters
\begin{eqnarray}
 \alpha_1^N &=& \nu - \beta_1^N = \alpha_1+\beta_1 + N - \beta_1 - \sum_{n=1}^N u_1^n \nonumber \\
 &=& \alpha_1+ N - \sum_{n=1}^N u_1^n \nonumber \\
 &=& \alpha_1+ \sum_{n=1}^N x_1 \\
 \forall i=2,\ldots,K-1: \beta_i^N &=& \beta_i + \sum_{n=1}^N u_i^n \\
 \alpha_i^N &=& c_i - \beta_i^N + \beta_{i-1}^N \nonumber \\
 &=& c_i - \beta_i + \beta_{i-1} + \sum_{n=1}^N (u_{i-1}^n-u_i^n) \nonumber \\
 &=& c_i + \beta_{i-1} - \beta_i +\sum_{n=1}^N x_i^n \nonumber \\
 &=& \alpha_i + \sum_{n=1}^N x_i^n
\end{eqnarray}
Hence, the meaning of \alphav is preserved.

\begin{table}[h!]
\centering
In this table, let $$\gamma_k=1+\exp(\eta_k)(1+\exp(\eta_{k+1})(1+\ldots(1+\exp(\eta_{K-1}))\ldots))$$

\begin{tabular}{c|c}
 \multicolumn{2}{c}{\textbf{multinomial stick-breaking distribution}} \\ \hline \hline
 standard form & $\prod_{k=1}^{K-1} v_k^{x_k} \left( 1- v_k \right)^{\sum_{i=k+1}^K x_i}$ \\ \hline
 constraints $\xv$ & $x_k \in \{0,1\}$, $\sum_{k=1}^K x_k=1$ \\ \hline
 constraints $\vv$ & $v_k \in [0,1]$ \\ \hline
 $\uv(\xv)$ & $\forall k=1,\ldots,K-1: u_k=\sum_{i=k+1}^K x_i$ \\ \hline
 $\etav$ &  $\eta_{K-1} = \log\left( \frac{1-v_{K-1}}{v_{K-1}}\right)$, $\forall k<K-1: \eta_{k}=\log\left( \frac{1-v_{k}}{v_{k}} v_{k+1}\right)$ \\ \hline
 $\vv$ & $v_{K-1}=\frac{1}{1+\exp(\eta_{K-1})}$, $\forall k<K-1: v_i = \frac{1}{1+\frac{\exp(\eta_i)}{v_{i+1}}}$ \\ \hline
 constraints $\etav$ & $\eta_k \in \mathbb{R}$ \\ \hline
 $g(\etav)$ & $\frac{1}{\gamma_1}=v_1$ \\ \hline
 $h(x)$ & 1 \\ \hline
 $\langle \uv(\xv) \rangle$ & $\langle u_k \rangle = \frac{\exp(\sum_{i=1}^k \eta_i) \gamma_{k+1}}{\gamma_1} = \prod_{i=1}^k (1-v_i)$ \\ \hline
 \multirow{2}{*}{$\mbox{Cov}(\uv(\xv))$} & $C_{kl} =\frac{\exp\left(\sum_{i=1}^{max(k,l)} \eta_i \right)\gamma_{max(k,l)+1}}{\gamma_1}-\frac{\exp\left(\sum_{i=1}^k \eta_i\right) \gamma_{k+1} \exp\left(\sum_{i=1}^k \eta_i\right)\gamma_{l+1}}{\gamma_1^2}$ \\
  & $= \prod_{i=1}^{max(k,l)} (1-v_i) - \prod_{i=1}^{k} (1-v_i) \prod_{j=1}^{l} (1-v_j) $
\end{tabular}
\caption{The stick-breaking distribution for multinomial variables}
\end{table}

\begin{table}[h!]
\centering

\hspace*{-2cm}
\begin{tabular}{c|c}
 \multicolumn{2}{c}{\textbf{stick-breaking prior}} \\ \hline \hline
 standard form & $\prod_{i=1}^{K-1} \frac{1}{B(\alpha_i,\beta_i)} v_i^{\alpha_i-1} (1-v_i)^{\beta_i-1} $ \\ \hline
 constraints $\alphav,\betav$ & $\alpha_k,\beta_k \in \mathbb{R}^+$ \\ \hline
 $\lambdav$ & $\frac{\betav}{\nu}$ \\ \hline
 $\nu$ & $\alpha_1+\beta_1$  \\ \hline
 constraints $\cv$ & $\forall i=2,\ldots,K-1:\, c_i>\beta_i-\beta_{i-1}$ \\ \hline
 $\alphav$ & $\alpha_1=\nu(1-\lambda_1)$, $\forall i=2,\ldots,K-1:\,\alpha_i=c_i-\beta_i+\beta_{i-1}$ \\ \hline
 constraints $\nu$ & $\nu \in \mathbb{R}^+$ \\ \hline
 constraints $\lambdav$ & $\lambda_i \in \mathbb{R}^+$ \\ \hline
 $f(\lambdav,\nu)$ & $\frac{\Gamma(\nu)}{\Gamma(\nu\lambda_1)\Gamma(\nu(1-\lambda_1))} \prod_{i=2}^{K-1} \frac{\Gamma(c_i+\nu \lambda_{i-1})}{\Gamma(\nu \lambda_i) \Gamma(c_i-\nu\lambda_i + \nu \lambda_{i-1})}$ \\ \hline
 $m(\etav)$ & $\prod_{i=2}^{K-1} v_i(\etav)^{c_i}$ \\ \hline
 \multirow{2}{*}{$\forall i<K-1: \langle \eta_i \rangle$} & $\psi\left(\nu\lambda_i\right)-\psi\left(c_i-\nu\lambda_i +\nu\lambda_{i-1}\right)+\psi\left(c_{i+1}-\nu\lambda_{i+1}+\nu\lambda_i \right)-\psi\left(c_{i+1}+\nu\lambda_i\right)$ \\
 & $=\psi\left(\beta_i\right)- \psi\left(\alpha_i \right) +\psi\left(\alpha_{i+1}\right)- \psi\left(\alpha_{i+1}+\beta_{i+1}\right) $ \\ \hline
 $\langle \eta_{K-1} \rangle$ & $\psi\left(\nu\lambda_i\right)-\psi\left(c_i-\nu\lambda_i +\nu\lambda_{i-1}\right) =\psi\left(\beta_i\right)- \psi\left(\alpha_i \right)$ \\ \hline
 \multirow{2}{*}{$\mbox{Var}(\eta_1)$} & $\psi^\prime\left(\nu\lambda_1 \right)+\psi^\prime\left(\nu(1-\lambda_1)\right)-\psi^\prime\left(c_2+\nu\lambda_1\right)+\psi^\prime\left(c_2-\nu\lambda_2+\nu\lambda_1 \right)$ \\
 & $ = \psi^\prime\left(\beta_1 \right)+\psi^\prime\left(\alpha_1\right)-\psi^\prime\left(\alpha_2+\beta_2\right)+\psi^\prime\left(\alpha_2 \right)$ \\ \hline
 \multirow{2}{*}{$\forall 1<i<K-1: \mbox{Var}(\eta_i)$} & $\psi^\prime\left(\nu\lambda_i \right)+\psi^\prime\left(c_i-\nu\lambda_i+\nu\lambda_{i-1}\right)-\psi^\prime\left(c_{i+1}+\nu\lambda_i\right)+\psi^\prime\left(c_{i+1}-\nu\lambda_{i+1}+\nu\lambda_i \right)$ \\
 & $=\psi^\prime\left(\beta_i \right)+\psi^\prime\left(\alpha_i \right)-\psi^\prime\left(\alpha_{i+1}+\beta_{i+1} \right)+\psi^\prime\left(\alpha_{i+1} \right)$ \\ \hline
 $\mbox{Cov}(\eta_i,\eta_{i+1})$ & $\psi^\prime\left(c_{i+1}-\nu\lambda_{i+1}+\nu\lambda_i \right) = \psi^\prime\left(\alpha_{i+1} \right)$ \\ \hline
 $\mbox{Var}(\eta_{K-1})$ & $ \psi^\prime\left(\nu\lambda_{K-1} \right)+\psi^\prime\left(c_{K-1}-\nu\lambda_{K-1}+\nu\lambda_{K-2} \right) = \psi^\prime\left(\beta_{K-1} \right)+\psi^\prime\left(\alpha_{K-1} \right) $ \\ \hline
 \multirow{2}{*}{$\frac{\partial \log(f(\lambdav,\nu))}{\partial \nu}$} & $\psi(\nu)-\lambda_1\psi(\nu\lambda_1)-(1-\lambda_1)\psi\left(\nu(1-\lambda_1)\right)$ \\
 & $+\sum_{i=2}^{K-1}\left[\lambda_{i-1}\psi(c_i+\nu\lambda_{i-1})-\lambda_i\psi(\nu\lambda_i)-(\lambda_{i+1}-\lambda_i)\psi(c_i-\nu\lambda_i+\nu\lambda_{i+1}) \right]$ \\ \hline
 \multirow{2}{*}{$\frac{\partial^2 \log(f(\lambdav,\nu))}{\partial \nu^2}$} & $\psi^\prime(\nu)-\lambda_1^2\psi^\prime(\nu\lambda_1)-(1-\lambda_1)^2\psi^\prime\left(\nu(1-\lambda_1)\right)$ \\
 & $+\sum_{i=2}^{K-1}\left[\lambda_{i-1}^2\psi^\prime(c_i+\nu\lambda_{i-1})-\lambda_i^2\psi^\prime(\nu\lambda_i)-(\lambda_{i+1}-\lambda_i)^2\psi^\prime(c_i-\nu\lambda_i+\nu\lambda_{i+1}) \right]$ \\ \hline
 $\langle g(\etav) \rangle$ & $1-\lambda_1=\frac{\alpha_1}{\nu} $ \\ \hline
 $\mbox{Var}(g(\eta))$ & $\frac{\lambda_1(1-\lambda_1)}{nu+1}=\frac{\alpha_1\beta_1}{\nu^2(1-\nu)}$ \\ \hline
 $p(\xv|\lambdav,\nu)$ & 
 {\setstretch{1.5}
  $\begin{array}{ccc} 
   x_1=1&:& 1-\lambda_1=\frac{\alpha_1}{\alpha_1+\beta_1} \\  
   x_{i\in\{2;\ldots;K-2\}}=1&:& \lambda_1\prod_{j=2}^{i-1} \frac{\nu\lambda_j}{c_j+\nu\lambda_{j-1}} \frac{c_i-\nu\lambda_i+\nu\lambda_{i-1}}{c_i+\nu\lambda_{i-1}} = \sum_{j=1}^{i-1} \frac{\beta_j}{\alpha_j+\beta_j} \frac{\alpha_i}{\alpha_j+\beta_j} \\
   x_{K}=1&:& \lambda_1 \prod_{j=2}^{K-1} \frac{\nu\lambda_j}{c_j+\nu\lambda_{j-1}} = \prod_{j=1}^{K-1} \frac{\beta_j}{\alpha_j+\beta_j}
  \end{array}$
}
\end{tabular}
\caption{The conjugate prior on the stick-breaking distribution for multinomial variables}
\end{table}

\clearpage

\subsection{Poisson-Gamma}

The Poisson distribution is a distribution over a univariate integer-valued random variable $x$, e.g. spike count or radioactive decay events. In standard form, it is given by
\begin{equation}
 p(x|r) = \frac{r^x \exp(-r)}{x!}
\end{equation}
where $r \in \mathbb{R}^+_0$ is the rate. Its sufficient statistic and natural parameter are 
\begin{eqnarray}
 u(x) &=& x \\
 \eta&=& \log(r)
\end{eqnarray}
and hence
\begin{equation}
 p(x|\eta) = \underbrace{\frac{1}{\Gamma(x+1)}}_{h(x)} \underbrace{\exp(-\exp(\eta))}_{g(\eta)} \exp(\eta\, u(x)).
\end{equation}

\noindent
Note that for $x \in \mathbb{N}$, $\Gamma(x+1) = x!$.
The conjugate prior on $r$ is a Gamma distribution with density
\begin{equation}
 p(r|\alpha,S) = \frac{1}{\Gamma(\alpha) S^\alpha} r^{\alpha-1} \exp\left(-\frac{r}{S} \right)
\end{equation}
where $\alpha \in \mathbb{R}^+_0$ is the shape parameter, and $S \in \mathbb{R}^+$ is the scale. To tranform this into exponential family form, let
\begin{eqnarray}
 \nu &=& \frac{1}{S}
 \lambda=\alpha S
\end{eqnarray}
and note that $\left|\frac{dr(\eta)}{d\eta} \right|=\exp(\eta)=r(\eta)$. Thus, we find
\begin{eqnarray}
 p(\eta|\nu,\lambda) &=& \frac{\nu^{\nu\lambda}}{\Gamma(\nu \lambda)} r(\eta)^{\nu\lambda-1} \exp\left( -\nu r(\eta) \right) r(\eta) \nonumber \\
 &=& \frac{\nu^{\nu\lambda}}{\Gamma(\nu \lambda)} \exp\left(\nu\lambda \log(r(\eta))\right) \exp\left( -\nu r(\eta) \right)  \nonumber \\
 &=& \underbrace{\frac{\nu^{\nu\lambda}}{\Gamma(\nu \lambda)}}_{f(\lambda,\nu)} \underbrace{\exp(- \exp(\eta)))^\nu}_{g(\eta)^\nu} \exp(\nu \lambda \eta )
\end{eqnarray}

\begin{table}[h!]
\centering
\begin{tabular}{c|c}
 \multicolumn{2}{c}{\textbf{Poisson distribution}} \\ \hline \hline
 standard form & $\frac{r^x \exp(-r)}{x!} $ \\ \hline
 constraints & $x \in \mathbb{N}_0$, $r \in \mathbb{R}_0^+$ \\ \hline
 $u(x)$ & $x$ \\ \hline
 $\eta$ & $\log(r)$ \\ \hline
 constraints $\eta$ & $\eta \in \mathbb{R}$ \\ \hline
 $g(\eta)$ & $\exp\left(-\exp(\eta)\right)$ \\ \hline
 $h(x)$ & $\frac{1}{\Gamma(x+1)}$ \\ \hline
 $\langle u(x) \rangle$ & $\exp(\eta) = r$ \\ \hline
 $\mbox{Var}(u(x))$ & $\exp(\eta)=r$
\end{tabular}
\vspace*{0.2cm}

\begin{tabular}{c|c}
 \multicolumn{2}{c}{\textbf{Gamma prior for Poisson-distributed RV }} \\ \hline \hline
 standard form & $\frac{1}{\Gamma(\alpha)S^\alpha} r^{\alpha-1} \exp\left(-\frac{r}{S}\right)$ \\ \hline
 constraints & $\alpha \in \mathbb{R}^+_0$, $S \in \mathbb{R}^+$ \\ \hline
 $\lambda$ & $\alpha S$ \\ \hline
 $\nu$ & $\frac{1}{S}$  \\ \hline
 constraints & $\nu \in \mathbb{R}^+$, $\lambda \in \mathbb{R}^+_0$ \\ \hline
 $f(\lambda,\nu)$ & $\frac{\nu^{\nu\lambda}}{\Gamma(\nu \lambda)}$ \\ \hline
 $m(\eta)$ & 1 \\ \hline
 $\langle \etav \rangle$ & $\psi(\nu\lambda) - \log(\nu) $ \\ \hline
 $\mbox{Var}(\eta)$ & $\psi^\prime(\nu\lambda)$ \\ \hline
 $\frac{\partial \log(f(\lambdav,\nu))}{\partial \nu}$ & $\lambda\left(\log(\nu)+1-\psi(\nu\lambda)\right) $ \\ \hline
 $\frac{\partial^2 \log(f(\lambdav,\nu))}{\partial \nu^2}$ & $\lambda\left(\frac{1}{\nu}-\lambda\psi^\prime(\nu \lambda) \right)$ \\ \hline
 $\langle g(\etav) \rangle$ & $\left(1+\frac{1}{\nu}\right)^{-\lambda\nu} $ \\ \hline
 $\mbox{Var}(g(\etav))$ & $\left(1+\frac{2}{\nu}\right)^{-\lambda\nu} - \left(1+\frac{1}{\nu}\right)^{-2\lambda\nu} $ \\ \hline
 $p(\xv|\lambdav,\nu)$ & $ \frac{\Gamma(\nu\lambda+x)}{\Gamma(\nu\lambda)\Gamma(x+1)} \frac{(\nu+1)^{-(\nu\lambda+x)}}{\nu^{-\lambda\nu}}$ \\ \hline
 $\langle u(x) \rangle_{p(x|\lambda,\nu)}$ & $\lambda$ 
\end{tabular}
\caption{Poisson distribution and conjugate Gamma prior}
\end{table}



\subsection{Multivariate Gaussian with Gauss-Wishart prior}

The multivariate Gaussian is widely used, e.g. all finite-sized marginals of a Gaussian process are Gaussian. But it 
is also a standard ingredient in parametric models for regression, e.g. linear or other basis function. In the standard form,
a multivariate Gaussian density of a vector-valued random variable with variates $\xv$, $dim(\xv)=D$
is parameterized by a mean vector $\muv$ and a symmetric, positive definite covariance matrix $\sigm$:

\begin{equation}
 p(\xv|\muv,\sigm) = \frac{1}{\sqrt{2\pi}^D \sqrt{|\sigm|}} \exp\left(-0.5 \cdot (\xv-\muv)^T\sigm^{-1}(\xv-\muv) \right)
\end{equation}

\noindent
It is often convenient to use the inverse of \sigm, called \textit{precision matrix} $\Pm = \sigm^{-1}$, which is also symmetric ($\Pm=\Pm^T)$ and positive definite:

\begin{equation}
 \label{gaussp}
 p(\xv|\muv,\Pm) = \frac{\sqrt{|\Pm|}}{\sqrt{2\pi}^D} \exp\left(-\frac{1}{2} \cdot (\xv-\muv)^T\Pm(\xv-\muv) \right)
\end{equation}

\noindent
To transform the Gaussian into the exponential family normal form, we rewrite the exponent as

\begin{equation}
 \label{expo}
 -\frac{1}{2} \cdot (\xv-\muv)^T\Pm(\xv-\muv) = -\frac{1}{2} \xv^T \Pm \xv + \muv^T \Pm \xv - \frac{1}{2} \muv^T \Pm \muv
\end{equation}
Note that 
\begin{eqnarray}
 \label{expo1}
 \xv^T \Pm \xv &=& \sum_i \sum_j \xv_i \Pm_{i,j} \xv_j = \sum_i \Pm_{i,i} \xv_i^2 + 2 \sum_i \sum_{j<i} \Pm_{i,j} \xv_i \xv_j \nonumber \\
 \label{expo2}
 \muv^T \Pm \xv &=& \sum_i (\Pm \muv)_i \xv_i
\end{eqnarray}
We therefore introduce sufficient statistics and natural parameters comprised of three parts: first, from the diagonal elements in eqn. \ref{expo1}, the row vectors
\begin{eqnarray}
\uv_d&=&(\xv_1^2,\xv_2^2,\ldots,\xv_D^2)^T \\
\label{natpargauss1}
\etav_d&=&-\frac{1}{2}(\Pm_{1,1},\Pm_{2,2},\ldots,\Pm_{d,d})^T.
\end{eqnarray}
Second, we order the off-diagonal elements (with $i<j$) in some arbitrary fashion (e.g. lectically) and construct the vectors
\begin{eqnarray}
\uv_c&=&(\xv_2\xv_1,\xv_3\xv_1,\xv_3\xv_2,\ldots,\xv_D\xv_{D-1})^T=\mbox{lt}(\xv\xv^T) \\
\label{natpargauss2}
\etav_c&=&-(\Pm_{2,1},\Pm_{3,1},\Pm_{3,2},\ldots,\Pm_{D,D-1})^T=-\mbox{lt}(\Pm)
\end{eqnarray}
i.e. $\etav_c$ contains the lower triangle of $\Pm$, and the $\mbox{lt}()$ operator extracts the lower triangle of a matrix, excluding the diagonal.
Third, from eqn. \ref{expo2}:
\begin{eqnarray}
 \uv_\mu &=& \xv \\
 \label{natpargauss3}
 \etav_\mu &=& \Pm \muv.
\end{eqnarray}
We stack these vectors into the total sufficient statistics and natural parameters
\begin{eqnarray}
 \uv(\xv) &=& \left( \begin{array}{c} \uv_d \\ \uv_c \\ \uv_\mu\end{array}\right) \\
 \etav &=& \left( \begin{array}{c} \etav_d \\ \etav_c \\ \etav_\mu \end{array} \right).
\end{eqnarray}
Noting that $\muv^T \Pm \muv = \muv^T \Pm^T \Pm^{-1} \Pm \muv = \etav_\mu^T \Pm^{-1} \etav_\mu$ and $\Pm=\Pm(\etav)$, eqn. \ref{expo} can be written as
\begin{equation}
 \etav^T \uv(\xv) - \frac{1}{2} \muv^t \Pm \muv= \underbrace{\etav_d^T \uv_d + \etav_c^T \uv_c}_{-\frac{1}{2} \xv^T \Pm \xv} + \underbrace{\etav_\mu^T \uv_\mu}_{\mu^T \Pm \xv} - \frac{1}{2} \etav_\mu^T \Pm(\etav)^{-1} \etav_\mu
\end{equation}
With these substitutions, the exponential family normal form of the multivariate Gaussian is therefore
\begin{equation}
 \label{gaussexpofam}
 p(\xv|\etav) = \underbrace{\frac{\sqrt{|\Pm(\etav)|}}{\sqrt{2\pi}^D} \exp\left(-\frac{1}{2} \etav_\mu^T \Pm(\etav)^{-1} \etav_\mu \right)}_{=:g(\etav)} \exp\left(\etav^T \uv(\xv) \right)
\end{equation}
and thus
\begin{equation}
 \log(g(\etav)) = -\frac{D}{2}\log(2\pi) + \frac{1}{2} \log\left(|\Pm(\etav)|\right) - \frac{1}{2} \etav_\mu^T \Pm(\etav)^{-1} \etav_\mu.
\end{equation}
To compute moments, we need the gradient of this expression w.r.t. $\etav$, whose components can be computed via the chain rule. Since
 \begin{eqnarray}
  \frac{\partial \Pm_{i,j}}{\partial (\etav_d)_k} &=& -2\, \delta_{i,k}\delta_{j,k} \\
  \frac{\partial \Pm_{i,j}}{\partial (\etav_c)_{(k,l)}} &=& -\delta_{i,k}\delta_{j,l} - \delta_{i,l}\delta_{j,k}
 \end{eqnarray}
 and $\Pm$ does not depend on $\etav_\mu$, we find
 \hspace*{-1.0cm}
 \begin{eqnarray}
  \frac{\partial \log(g(\etav))}{\partial (\etav_\mu)_k} &=& -\frac{1}{2} \cdot 2 (\Pm^{-1} \etav_\mu)_k = - \muv_k \\
  \frac{\partial \log(g(\etav))}{\partial (\etav_d)_k} &=& \sum_{i,j} \frac{1}{2} \left[\Pm^{-1}_{i,j} + \Pm^{-1}\etav_\mu\etav_\mu^T\Pm^{-1}\right] \cdot (-2) \delta_{i,k}\delta_{j,k} = -\Pm^{-1}_{k,k}-\muv_k^2 \\ 
  \frac{\partial \log(g(\etav))}{\partial (\etav_c)_{(k,l)}} &=& \sum_{i,j} \frac{1}{2} \left[ \Pm^{-1}_{i,j}+ \Pm^{-1}\etav_\mu\etav_\mu^T\Pm^{-1}\right] \cdot (-\delta_{i,k}\delta_{j,l} - \delta_{i,l}\delta_{j,k}) = -\Pm^{-1}_{k,l}-\muv_k\muv_l \nonumber \\
 \end{eqnarray}
which implies, by virtue of eqn. \ref{expectation_u}:
\begin{eqnarray}
 \langle \uv_\mu(\xv) \rangle &=& \muv \\
 \langle \uv_d(\xv) \rangle &=& -\mbox{diag}(\Pm^{-1})-\muv^T\muv \\
 \langle \uv_c(\xv)_{(kl)} \rangle &=& -\Pm^{-1}_{k,l}-\muv_k\muv_l
\end{eqnarray}
i.e. the well-known expressions for expectations of Gaussians. The second derivatives (necessary for the evaluation of the gradient of the KL divergence) are omitted here, they can be computed by automatic differentiation
from the above expressions, e.g. by \texttt{Theano}.

The prior on the parameters of the Gaussian is given by a (multivariate) Gauss-Wishart distribution \cite{BishopMachLearn}. In standard form, it is given by

\begin{eqnarray}
 \label{gausswishart}
 p(\muv,\Pm|\beta,\munv,\gamma,\Vm) &=& p(\muv|\beta,\Pm) p(\Pm|\gamma,\Vm) \\
 p(\muv|\beta,\Pm) &=& \frac{\beta^{\frac{D}{2}} |\Pm|^{\frac{1}{2}}}{\sqrt{2\pi}^D} \exp\left(-\frac{1}{2} \beta (\muv-\munv)^T \Pm (\muv-\munv) \right) \\
 p(\Pm|\gamma,\Vm) &=& \frac{|\Pm|^{\frac{\gamma-D-1}{2}}}{2^{\frac{\gamma D}{2}} |\Vm|^{\frac{\gamma}{2}} \Gamma_D\left(\frac{\gamma}{2}\right)} \exp\left(-\frac{1}{2} \tr(\Vm^{-1}\Pm) \right)
\end{eqnarray}
where $\Gamma_D(x)$ is the multivariate gamma function \cite{MultivarGamma}:
\begin{equation}
 \Gamma_D(x)=\pi^{\frac{D(D-1)}{4}} \prod_{j=1}^D \Gamma\left(x+\frac{1-j}{2}\right)
\end{equation}

We reparameterize as follows:
\begin{eqnarray}
 \label{gwparm1}
 \beta &=& \nu \\
 \gamma &=& \nu+\alpha, \; \alpha > 0 \;\mbox{and const.}\\
 \lambdav_\mu &=& \munv \\
 \Bm &=& \Vm^{-1} \\
 (\lambdav_d)_i &=& \left(\frac{B_{i,i}}{\nu} + (\munv)_i^2 \right) \\
 (\lambdav_c)_{(k,l)} &=& \frac{B_{k,l}}{\nu}+(\munv)_k (\munv)_l = \frac{B_{l,k}}{\nu}+(\munv)_k (\munv)_l \\
 \label{gwparm7}
 \lambdav &=& \left(\lambdav_\mu,\lambdav_d,\lambdav_c \right)^T
\end{eqnarray}
Using these substitutions and $\muv^T \Pm \muv = \etav_\mu^T \Pm^{-1} \etav_\mu$, the arguments of the exponentials in eqn. \ref{gausswishart} can be written as
\begin{eqnarray}
 && -\nu \frac{1}{2} \muv^T \Pm \muv + \nu \munv^T \Pm \muv - \nu \frac{1}{2} \munv^T \Pm \munv - \frac{1}{2} \tr(\Bm \Pm) \nonumber \\
 &=& \nu \left( -\frac{1}{2} \etav_\mu^T \Pm^{-1} \etav_\mu + \etav_\mu^T \munv -\frac{1}{2} \tr\left(\Pm\left(\frac{\Bm}{\nu}+\munv\munv^T\right)\right) \right) \nonumber \\
 &=& \nu \left( -\frac{1}{2} \etav_\mu^T \Pm^{-1} \etav_\mu + \etav_\mu^T \lambdav_\mu + \etav_d^T \lambdav_ds + \etav_c^T \lambdav_c \right)
\end{eqnarray}
Thus eqn. \ref{gausswishart} becomes
\begin{eqnarray}
 \label{gausswishart2}
 p(\muv,\Pm|\nu,\munv,\Bm) &=& \sqrt{2\pi}^{(\nu-1)D} \frac{\nu^{-\frac{D}{2}}|B(\lambdav)|^{\frac{\nu+\alpha}{2}}}{2^{\frac{(\nu+\alpha)D}{2}}\Gamma_D\left(\frac{\nu+\alpha}{2}\right)} |P|^{\frac{\alpha-D}{2}} \nonumber \\
  &\times& \left(\frac{1}{\sqrt{2\pi}^D}\right)^\nu |P|^{\frac{\nu}{2}} \exp\left(-\frac{1}{2} \etav_\mu^T \Pm^{-1} \etav_\mu \right)^\nu \nonumber \\
  &\times& \exp\left( \nu \etav_\mu^T \lambdav_\mu + \nu \etav_d^T \lambdav_d + \nu \etav_c^T \lambdav_c \right).
\end{eqnarray}
To transform this into the desired exponential family form, we need to change $\muv,\Pm$ into the natural parameters of the multivariate Gaussian (see eqns. \ref{natpargauss1},\ref{natpargauss2},\ref{natpargauss3}).
This can be accomplished by multiplying eqn.\ref{gausswishart} with the determinant of the Jacobian $\mathbf{J}$ of that transformation, which can be constructed as a block matrix as follows: stack the diagonal elements of $\Pm$
into the vector $\Pm_d$ of dimensionality $D$, the off-diagonal lower triangle into the vector $\Pm_c$ of dimensionality $Q=\frac{D(D-1)}{2}$.
The Jacobian then has the following structure, which follows from the definitions in eqns. \ref{natpargauss1},\ref{natpargauss2},\ref{natpargauss3} 
(rows and columns labelled with variable names):
\begin{equation}
 \mathbf{J} = \begin{array}{c||ccc|ccc|ccc}
		  & \muv_1 & \ldots & \muv_D & (\Pm_d)_1 & \ldots & (\Pm_d)_D & (\Pm_c)_1 & \ldots & (\Pm_c)_Q \\ \hline\hline
(\etav_\mu)_1	  &        &        &        &           &        &           &           &        &           \\
\vdots            &        &\Pm^{-1}&        &           &\mathbf{0}&         &           & \mathbf{0}&        \\
(\etav_\mu)_D     &        &        &        &           &        &           &           &           &        \\ \hline
(\etav_d)_1       &        &        &        &           &        &           &           &           &        \\ 
\vdots            &        &\frac{\partial \muv}{\partial \etav_d}&&&-2\cdot\mathbf{1}_D&&& \mathbf{0}&        \\
(\etav_d)_D       &        &        &        &           &        &             \\
(\etav_c)_1       &        &        &        &           &        &             \\
\vdots            &        &\frac{\partial \muv}{\partial \etav_c}&&&\mathbf{0}&&         &-\mathbf{1}_Q&          \\
(\etav_c)_Q       &        &   &        &           &        &             \\
	      \end{array}
\end{equation}
By virtue of eqns.  397 and 398 in \cite{MatrixCook}, the absolute value of the determinant of $\mathbf{J}$ is therefore
\begin{equation}
 |J| = 2^D |P^{-1}(\etav)|
\end{equation}
Reparameterizing eqn. \ref{gausswishart2} in terms of $\etav$ and multiplying with this expression yields (cf. eqn. \ref{gaussexpofam} for the definition of $g(\etav)$):
\begin{eqnarray}
 \label{gausswishart3}
 p(\etav|\nu,\munv,\Bm) &=& \underbrace{2^D \sqrt{2\pi}^{(\nu-1)D} \frac{\nu^{\frac{D}{2}}|\Bm(\lambdav)|^{\frac{\nu+\alpha}{2}}}{2^{\frac{(\nu+\alpha)D}{2}}\Gamma_D\left(\frac{\nu+\alpha}{2}\right)}}_{=:f(\nu,\lambdav)} \underbrace{|\Pm|^{\frac{\alpha-D}{2}-1}}_{=:m(\etav)} \nonumber \\
  &\times& \underbrace{\left(\frac{1}{\sqrt{2\pi}^D}\right)^\nu |P|^{\frac{\nu}{2}} \exp\left(-\frac{1}{2} \etav_\mu^T \Pm^{-1} \etav_\mu \right)^\nu}_{=g(\etav)^\nu}\ \nonumber \\
  &\times& \exp\left( \nu \etav_\mu^T \lambdav_\mu + \nu \etav_d^T \lambdav_d + \nu \etav_c^T \lambdav_c \right) \nonumber \\
  &=& f(\nu,\lambdav) m(\etav) g(\etav)^\nu \exp(\nu \etav^T \lambdav)
\end{eqnarray}
which is the exponential family normal form of the Gauss-Wishart prior on the parameters of the multivariate Gaussian.

To calculate expectations, we need the derivatives of $\log(f(\nu,\lambda))$:
\begin{equation}
 \log(f(\nu,\lambda)) = C+\frac{D\nu}{2} \log(2\pi) + \frac{D}{2}\log(\nu) - \log\left(\Gamma_D\left(\frac{\nu+\alpha}{2}\right)\right) + \frac{\nu+\alpha}{2}\log(|\Bm(\lambdav)|)
\end{equation}
with $C=D\log(2) - \frac{D}{2}\log(2\pi) - \frac{\alpha D}{2} \log(2)$. Thus
\begin{equation}
 \frac{\partial f(\nu,\lambda)}{ \partial \nu} = \frac{D}{2} \log(2\pi) + \frac{D}{2\nu} - \frac{1}{2} \Psi_D\left(\frac{\nu+\alpha}{2}\right) + \frac{1}{2}\log(|\Bm(\lambdav)|)
\end{equation}
where $\Psi_D(x) = \frac{\partial \log(\Gamma_D(x))}{\partial x}$ is the multivariate digamma function.
For the derivatives w.r.t. $\lambdav$, note that eqns. \ref{gwparm1}-\ref{gwparm7} imply $B_{i,i}=\nu(2(\lambda_d)_i-(\lambda_mu)^2_i$ and $\forall i>j: B_ij = \nu((\lambda_c)_{i,j}-(\lambda_\mu)_i(\lambda_\mu)_j$,
$\forall j>i: B_ij = \nu((\lambda_c)_{j,i}-(\lambda_\mu)_i(\lambda_\mu)_j$, hence
\begin{eqnarray}
 \frac{\partial B_{i,j}}{\partial (\lambdav_\mu)_k} &=& - \nu ((\lambdav_\mu)_i \delta_{j,k} + (\lambdav_\mu)_j \delta_{i,k}) \\
 \frac{\partial B_{i,j}}{\partial (\lambdav_d)_k} &=& \nu \delta_{i,k}\delta_{j,k} \\
 \frac{\partial B_{i,j}}{\partial (\lambdav_c)_{k,l}} &=& \nu (\delta_{i,k}\delta_{j,l} + \delta_{i,l}\delta_{j,k})
\end{eqnarray}
With formula 57 from \cite{MatrixCook}, we therefore find
\begin{eqnarray}
 \frac{\partial \log(f(\nu,\lambdav))}{\partial (\lambdav_d)_k} &=& \sum_{m,n} \frac{\nu+\alpha}{2} \frac{\partial \log(|\Bm|)}{\partial B_{m,n}} \frac{\partial B_{m,n}}{\partial (\lambdav_d)_k} \nonumber \\
 &=& \frac{\nu(\nu+\alpha)}{2}  \sum_{m,n} \Bm^{-1}_{m,n} \delta_{m,k} \delta_{n,k} = -\nu(\nu+\alpha) \Bm^{-1}_{k,k} \\
 \frac{\partial \log(f(\nu,\lambdav))}{\partial (\lambdav_c)_{k,l}} &=& \nu(\nu+\alpha) \Bm^{-1}_{k,l} \\
 \frac{\partial \log(f(\nu,\lambdav))}{\partial (\lambdav_\mu)_k} &=& -\nu \frac{(\nu+\alpha)}{2} \sum_{m,n} \Bm^{-1}_{m,n} ((\lambda_\mu)_m \delta_{n,k}+(\lambda_\mu)_n \delta_{m,k}) \nonumber \\
 &=& -\nu(\nu+\alpha) \sum_n \Bm^{-1}_{k,n} (\lambdav_\mu)_n
\end{eqnarray}
The expectations of the sufficient statistics of the multivariate Gaussian are thus ($\mbox{lt}(\Xv)$: vectorized form of lower triangle of $\Xv$)
\begin{eqnarray}
 \langle \etav_\mu \rangle &=& \langle \Pm \muv \rangle = (\nu+\alpha) \Bm^{-1}(\lambdav) \lambda_\mu = \gamma \Vm \munv \\
 \langle \etav_d \rangle &=& -\frac{1}{2}\langle \mbox{diag}(\Pm) \rangle = -\frac{(\nu+\alpha)}{2} \mbox{diag}(\Bm^{-1}) \\
 \langle \etav_c \rangle &=& -\langle \mbox{lt}(\Pm) \rangle = -(\nu+\alpha) \mbox{lt}(\Bm^{-1}) \\
 &\Rightarrow & \langle \Pm \rangle = \gamma \Vm
\end{eqnarray}
Second derivatives can be obtained similarly, or more easily by automatic differentiation.

The expectation of the sufficient statistics $\uv(\xv)$ can be computed from eqn. \ref{expected_u_under_prior}. We already computed $\lambdav$ (eqn. \ref{gwparm7}). 
The surface integral vanishes, because $p(\xv|\lambdav,\nu) \rightarrow 0$ as $\lambdav_\mu \to \infty$, since the density must be normalized. Furthermore, in the subspace
where $|\Pm|=0$, the density is also zero if $\gamma>D-1$. Since $m(\etav)$ does not depend on $\munv$, we see that
\begin{equation}
 \langle \uv_\mu(\xv) \rangle_{p(\xv|\lambdav,\muv)} = \lambdav_\mu = \munv
\end{equation}

\noindent
For $\uv_d, \; \uv_c$, it remains to evaluate the expectation
\begin{eqnarray}
 \langle \nabla_\etav \log(m(\etav)) \rangle_{p(\xv|\lambdav,\muv)} &=& -2 \left\langle \left(\frac{\alpha-D}{2}-1\right) \Pm^{-1}  \right\rangle_{p(\xv|\lambdav,\muv)} \nonumber \\
 &=& -(\alpha-D-2) \frac{\Vm^{-1}}{\nu+\alpha-D-1}
\end{eqnarray}
where the factor $-2$ results from the differentation of the elements of $\Pm$ w.r.t. $\etav$, see also the Jacobian above. The entries of $\Vm^{-1}$ have to be suitably rearranged to
match the entries of $\etav$.
Thus we find
\begin{eqnarray}
 \langle \uv_d(\xv) \rangle_{p(\xv|\lambdav,\muv)} &=& \mbox{diag}(\munv \munv^T) + \frac{\mbox{diag}(\Vm^{-1})}{\nu} - \frac{\alpha-D-2}{\nu}\frac{\mbox{diag}(\Vm^{-1})}{\nu+\alpha-D-1} \nonumber \\
 &=& \mbox{diag}(\munv \munv^T) + \frac{\nu+1}{\nu}\frac{\mbox{diag}(\Vm^{-1})}{\nu+\alpha-D-1}
\end{eqnarray}
and likewise for $\uv_c$.





\begin{table}[h!]
\centering
\begin{tabular}{c|c}
 \multicolumn{2}{c}{\textbf{Multivariate Gaussian distribution}} \\ \hline \hline
 standard form & $\frac{1}{\sqrt{2\pi}^D \sqrt{|\sigm|}} \exp\left(-0.5 \cdot (\xv-\muv)^T\sigm^{-1}(\xv-\muv) \right)$ \\ \hline
 constraints & $\xv,\muv \in \mathbb{R}^D$, $\sigm$ positive semidefinite and symmetric \\ \hline
 $\uv(\xv)$ & $\uv_d=\mbox{diag}(\xv\xv^T)$,$\uv_c=\mbox{lt}(\xv\xv^T)$,$\uv_\mu=\xv$ \\ \hline
 $\etav$ & $\etav_d=-\frac{1}{2}\mbox{diag}(\Pm)$, $\etav_c=-\mbox{lt}(\Pm)$, $\etav_\mu=\Pm \muv$ \\ \hline
 constraints $\eta$ & $\etav_\mu \in \mathbb{R}^D$, $\etav_d \leq 0$, $\etav_c$ s.t. $\Pm$ pos.semidef. \\ \hline
 $g(\etav)$ & $\frac{\sqrt{|\Pm(\etav)|}}{\sqrt{2\pi}^D} \exp\left(-\frac{1}{2} \etav_\mu^T \Pm(\etav)^{-1} \etav_\mu \right)$ \\ \hline
 $h(x)$ & 1 \\ \hline
 $\langle \uv(\xv) \rangle$ & $ \langle \uv_\mu \rangle = \muv,\,\langle \uv_d \rangle = -\mbox{diag}(\Pm^{-1}+\muv\muv^T),\,\langle \uv_c \rangle = -\mbox{lt}({\Pm^{-1}+\muv\muv^T})$
\end{tabular}
\vspace*{0.2cm}

\begin{tabular}{c|c}
 \multicolumn{2}{c}{\textbf{Gauss-Wishart prior for multivariate Gaussian RV }} \\ \hline \hline
 standard form & $\frac{\beta^{\frac{D}{2}} |P|^{\frac{1}{2}}}{\sqrt{2\pi}^D} \exp\left(-\frac{1}{2} \beta (\muv-\munv)^T \Pm (\muv-\munv) \right) \frac{|\Pm|^{\frac{\gamma-D-1}{2}}}{2^{\frac{\gamma D}{2}} |\Vm|^{\frac{\gamma}{2}} \Gamma_D\left(\frac{\gamma}{2}\right)} \exp\left(-\frac{1}{2} \tr(\Vm^{-1}\Pm) \right)$ \\ \hline
 constraints & $\muv,\munv \in \mathbb{R}^D$, $\Pm, \Vm$ pos.semidef., $\beta>0$,$\gamma>D-1$ \\ \hline
 $\lambdav$ & $\Bm=\Vm^{-1}$, $\lambdav_d= \mbox{diag}\left(\frac{\Bm}{\nu} + \munv\munv^T\right)$, $\lambdav_c= \mbox{lt}\left(\frac{\Bm}{\nu} + \munv\munv^T\right)$, $\lambdav_\mu= \munv$ \\ \hline
 $\nu$ & $\nu=\beta$, $\gamma=\nu+\alpha$  \\ \hline
 constraints & $\nu \in \mathbb{R}^+$, $\alpha$ const. and s.t. $\nu+\alpha>D-1$ \\ \hline
 $f(\nu,\lambdav)$ & $ 2^{\frac{D(1-\alpha)}{2}} \sqrt{\pi}^{(\nu-1)D} \frac{\nu^{-\frac{D}{2}}|\Bm(\lambdav)|^{\frac{\nu+\alpha}{2}}}{\Gamma_D\left(\frac{\nu+\alpha}{2}\right)} $ \\ \hline
 $m(\etav)$ & $|\Pm(\etav)|^{\frac{\alpha-D}{2}-1}$ \\ \hline
 $\langle \etav \rangle$ & $\langle \etav_\mu \rangle = (\nu+\alpha) \Bm^{-1}(\lambdav)\lambdav_\mu$, $\langle \etav_d \rangle = -(\nu+\alpha) \mbox{diag}(\Bm^{-1})$,  $\langle \etav_c \rangle = -(\nu+\alpha) \mbox{lt}(\Bm^{-1})$ \\ \hline
 $\frac{\partial \log(f(\lambdav,\nu))}{\partial \nu}$ & $\frac{D}{2} \log(\pi) - \frac{D}{2\nu} - \frac{1}{2} \Psi_D\left(\frac{\nu+\alpha}{2}\right) + \frac{1}{2}\log(|\Bm(\lambdav)|)$ \\ \hline
 $p(\xv|\lambdav,\nu)$ & $ \frac{\Gamma_D\left(\frac{\nu+\alpha+1}{2}\right)}{\sqrt{\pi}\Gamma_D\left(\frac{\nu+\alpha}{2}\right)} \left(\frac{\nu}{\nu+1}\right)^{\frac{D}{2}} \frac{|\Bm(\lambdav)|^{\frac{\nu+\alpha}{2}}}{\left|\Bm\left(\frac{\nu\lambdav+\uv(\xv)}{\nu+1}\right)\right|^{\frac{\nu+\alpha+1}{2}}} $ \\ \hline
 $\langle \uv_\mu (\xv) \rangle_{p(\xv|\lambdav,\mu)}$ & $\lambdav_u = \munv$ \\ \hline
 $\langle \uv_d (\xv) \rangle_{p(\xv|\lambdav,\mu})$ & $\lambdav_d - \frac{\alpha-D-2}{\nu}\frac{\mbox{diag}(\Vm^{-1})}{\nu+\alpha-D-1} = \mbox{diag}(\munv \munv^T) + \frac{\nu+1}{\nu} \mbox{diag}(\Vm^{-1})$ \\ \hline
 $\langle \uv_c (\xv) \rangle_{p(\xv|\lambdav,\mu})$ & $\lambdav_c - \frac{\alpha-D-2}{\nu}\frac{\mbox{lt}(\Vm^{-1})}{\nu+\alpha-D-1} = \mbox{lt}(\munv \munv^T) + \frac{\nu+1}{\nu} \mbox{lt}(\Vm^{-1})$
\end{tabular}
\caption{Multivariate Gaussian distribution and conjugate Gauss-Wishart prior}
\end{table}

\subsubsection{Univariate Gauss-Gauss-Gamma}

trivial, see above.

\section{Random identities that might be useful}

Decomposition of Kullback-Leibler divergence for multivariate Gaussians: let
\begin{equation}
 p(\xv_1,\xv_2|\muv_1,\muv_2,\kv_{11},\kv_{21},\kv_{22}) = \mathcal{N}\left((\xv_1,\xv_2)|(\muv_1,\muv_2),\left(\begin{array}{cc}\kv_{11} & \kv_{21}^T \\ \kv_{21} &\kv_{22} \end{array}\right)\right)
\end{equation}
be a joint multivariate Gaussian on $\xv_1,\xv_2$. The conditional distribution of $\xv_2$ given $\xv_1$ is then also multivariate Gaussian (see \cite{MatrixCook}):
\begin{eqnarray}
 \Mm_{21} &=& \kv_{21} \kv_{11}^{-1} \\
 \kv_{2|1} &=& \kv_{22} - \Mm_{21} \kv_{11}^{-1} \Mm_{21}^T \\
 p(\xv_1|\xv_2) &=& \mathcal{N}\left(\xv_2|\muv_2+\Mm_{21}(\xv_1-\muv_1),\kv_{2|1}\right)
\end{eqnarray}
Assume now we had a variational posterior for $\mv{x}_1,\mv{x}_2$, which we decompose in the following way (tilde indicates variational parameters):
\begin{equation}
 q(\xv_1,\xv_2) = q(\xv_2|\xv_1) q(\xv_1) = \mathcal{N}\left(\xv_2|\muvt_2+\Mmt_{21}\xv_1,\kvt_{2|1}\right) \mathcal{N}\left(\xv_1|\muvt_1,\kvt_{1}\right)
\end{equation}
which is a generalized form of the conditional Gaussian above, because $\muvt_{2|1}, \Mmt_{21},\kvt_{2|1}$ are now free variational parameters.
Note the following decomposition property of the KL-divergence, which follows directly from its definition:
\begin{equation}
 D\left(q(\xv_1,\xv_2)|p(\xv_1,\xv_2)\right) = \left\langle D\left(q(\xv_2|\xv_1)|p(\xv_2|\xv_1)\right)\right\rangle_{q(\xv1)} + D\left(q(\xv_1)|p(\xv_1)\right)
\end{equation}
Using the above distributions, the second term on the right hand side is given by the usual expression for the KL-divergence between multivariate Gaussians:
\begin{eqnarray}
 D\left(q(\xv_1)|p(\xv_1)\right) &=& \frac{1}{2} \left(\mbox{tr}\left[\kv_{1}^{-1}\kvt_1\right]+(\muvt_1-\muv1)^T \kv_1^{-1} (\muvt_1-\muv1) \right. \nonumber \\
 && \left. - \mbox{dim}[\xv_1] + \log(|\kv_1|) - \log(|\kvt_1|) \right)
\end{eqnarray}
and the first term is
\begin{eqnarray}
 && \left\langle D\left(q(\xv_2|\xv_1)|p(\xv_2|\xv_1)\right)\right\rangle_{q(\xv1)} \nonumber \\
  &=& \frac{1}{2} \left(\mbox{tr}\left[\kv_{2|1}^{-1}\kvt_{2|1}\right]+ \right. \nonumber \\
 && + \left(\muvt_2+\Mmt_{2|1}\muvt_1-(\muv_2+\Mm_{2|1}(\muvt_1-\muv_1))\right)^T \kv_1^{-1} \left(\muvt_2+\Mmt_{2|1}\muvt_1-(\muv_2+\Mm_{2|1}(\muvt_1-\muv_1))\right) \nonumber \\
 &&   - \mbox{dim}[\xv_2] + \log(|\kv_{2|1}|) - \log(|\kvt_{2|1}|) \nonumber \\
 && \left. + \mbox{tr}\left[(\Mmt_{2|1}-\Mm_{2|1})^T \kv_{2|1}^{-1}(\Mmt_{2|1}-\Mm_{2|1})\kvt_{2|1}  \right] \right)
\end{eqnarray}
This expression is zero iff: $\Mmt_{2|1}=\Mm_{2|1}, \kv_{2|1}=\kvt_{2|1}, \muvt_1=\muv_1, \muvt_2=\muv_2-\Mm_{2|1}\muv_1$. Note that a zero expectation can not be
achieved if we had made the Ansatz $q(\xv_2|\xv_1)=\mathcal{N}\left(\xv_2|\muvt_{2|1},\kvt_{2|1}\right)$, because the KL-divergence is positive, thus for its expectation to be zero,
it has to be zero for all values of $\xv_1$ which requires an explicit representation of the mean projection matrix \Mm.




\begin{thebibliography}{99}
 
\bibitem{BishopMachLearn}
C.M. Bishop.
\newblock {\em Pattern Recognition and Machine Learning}.
\newblock Springer, New York, 2007.

\bibitem{MatrixCook}
K. B. Petersen and M. S. Pedersen.
\newblock {\em The Matrix Cookbook}.
\newblock Version 20121115, http://www.math.uwaterloo.ca/~hwolkowi/matrixcookbook.pdf

\bibitem{MultivarGamma}
A.T. James. 
\newblock {\em Distributions of Matrix Variates and Latent Roots Derived from Normal Samples}. 
\newblock Ann. Math. Statist. 35 (1964), no. 2, 475--501.




\end{thebibliography}




\end{document}
