"""
Define parameter distributions for pySPACE nodes.
"""
from __future__ import division
import abc
import copy
import numpy
from scipy import stats
try:
# noinspection PyPackageRequirements
from matplotlib import pyplot as plt
from matplotlib import ticker as mtick
except ImportError:
plt = None
mtick = None
PARAMETER_ATTRIBUTE = "__hyperparameters"
[docs]class ParameterDecorator(object):
"""
Abstract base class for creating parameter decorators
to declare the as optimization parameters.
BE CAREFUL WHEN IMPLEMENTING NEW DECORATORS. THEY MUST BE
WRAPPED AND SUPPORTED BY __ALL__ OPTIMIZATION ALGORITHMS.
"""
__metaclass__ = abc.ABCMeta
[docs] def __init__(self, parameter_name):
"""
Create a new optimization parameter with `parameter_name` as name
The names of parameters have to be unique, as they get identified
by the name.
:param parameter_name: The name of the parameter to create.
:type parameter_name: str
"""
self.parameter_name = parameter_name
@abc.abstractmethod
[docs] def execute(self, class_, parameters):
"""
Execute the decorator
This method will be called during creation of the class object
and will update the given set of hyperparameters according
to the implementation of the subclass.
:param class_: The class object this parameter decorates
:type class_: type
:param parameters: The set of parameters to append to or delete from
:type parameters: set(ParameterDecorator)
"""
raise NotImplementedError("Execute Method has to be overwritten by subclasses")
@abc.abstractmethod
[docs] def plot(self):
"""
Plot the given parameter distribution
This method is used to plot the specified distribution of this decorator.
For plotting either (if installed) the `matplotlib.pyplot` can be used and return a figure,
or (if not installed) a string specifying the distribution can be returned.
:returns: The figure where this distribution is plotted or a string specifying the distribution
:rtype: matplotlib.figure.Figure | str
"""
raise NotImplementedError()
[docs] def __call__(self, class_):
if not hasattr(class_, PARAMETER_ATTRIBUTE):
# No hyper parameter attribute, create a new one
setattr(class_, PARAMETER_ATTRIBUTE, set())
# Deep copy the parameter attribute to avoid side-effects to super-classes
parameters = copy.deepcopy(getattr(class_, PARAMETER_ATTRIBUTE))
# Execute the Decorator on the copy
self.execute(class_, parameters)
# And replace the attribute with the copy
setattr(class_, PARAMETER_ATTRIBUTE, parameters)
# Return the class object
return class_
[docs] def __eq__(self, other):
if hasattr(other, "parameter_name"):
return self.parameter_name == other.parameter_name
elif isinstance(other, basestring):
return self.parameter_name == other
return False
[docs] def __ne__(self, other):
return not self.__eq__(other)
[docs] def __hash__(self):
return hash(self.parameter_name)
[docs] def __str__(self):
return self.parameter_name
[docs] def __repr__(self):
return "{cls}<{name}>".format(cls=self.__class__.__name__, name=self.parameter_name)
[docs]class AddedParameterDecorator(ParameterDecorator):
"""
This mixin adds an execute method to classes that inherit from it.
This mixin will check if the given parameter is already defined
and if so removes the old definition and adds the new one.
"""
__metaclass__ = abc.ABCMeta
[docs] def execute(self, class_, parameters):
if self in parameters:
parameters.remove(self)
parameters.add(self)
[docs]class QMixin(object):
"""
This mixin adds a regulation parameter `q`
to bind a distribution to discrete values.
"""
[docs] def __init__(self, q):
"""
Add the new regulation parameter.
:param q: The regulation value
:type q: float
"""
self.__q = q
@property
def q(self):
return self.__q
[docs] def round_to_q(self, value):
"""
Round a value to the next multiple of `q`.
This is required for the plotting only.
:param value: The value to round
:type value: float
"""
return numpy.round(value / self.q) * self.q
[docs] def calc_probability(self, value, pdf_func):
# Integral borders
a = value - self.q / 2
b = value + self.q / 2
# Create a linear space between these two borders
x, dx = numpy.linspace(start=a, stop=b, num=10000, retstep=True)
# Then calculate the PDF value for each of them
y = numpy.vectorize(lambda v: pdf_func(v))(x)
# And then integrate
return numpy.trapz(y=y, x=x, dx=dx)
[docs]class ChoiceParameter(AddedParameterDecorator):
"""
Defines a parameter as to be chosen from
the given set of options.
This parameter will be then chosen from this
set during the optimization.
..code-block:: python
>>> @ChoiceParameter("test", choices=["A", "B", "C"]
... class A(object):
... pass
"""
[docs] def __init__(self, parameter_name, choices):
"""
Create a new choice parameter with `parameter_name` as name
and `choices` as the possible values.
:param parameter_name: The name of the parameter to create.
:type parameter_name: str
:param choices: The possible values for this parameter.
:type choices: str | List[str]
"""
super(ChoiceParameter, self).__init__(parameter_name=parameter_name)
if not isinstance(choices, list):
choices = [choices]
self.__choices = choices
@property
def choices(self):
return self.__choices
[docs] def plot(self):
return "{self!r}(choices={self.choices!s})".format(self=self)
[docs]class BooleanParameter(ChoiceParameter):
"""
Defines a parameter as a being a boolean parameter.
This parameter will either be "true" or "false"
during the optimization.
..code-block:: python
>>> @BooleanParameter("test")
... class A(object):
... pass
"""
[docs] def __init__(self, parameter_name):
"""
Creates a new boolean parameter with `parameter_name` as name.
:param parameter_name: The name of the parameter to create.
:type parameter_name: str
"""
super(BooleanParameter, self).__init__(parameter_name, [True, False])
[docs]class PChoiceParameter(ChoiceParameter):
"""
Defines a parameter as a probability choice.
This parameter will sample each of the given
choices with the according probability.
..code-block:: python
>>> @PChoiceParameter("test", choices={"A": 0.5, "B": 0.25, "C": 0.25})
... class A(object):
... pass
"""
[docs] def __init__(self, parameter_name, choices):
"""
Create a new probability choice parameter
with `parameter_name` as name and `choices`
as the possible values.
Each choice must be a tuple containing first
the probability in range from 0 to 1 for that
choice and the value to choose as a second argument.
The probabilities of all choices need to sum up to 1.
:param parameter_name: The name of the parameter to create.
:type parameter_name: str
:param choices: A dictionary of tuples containing the value
of each choice as keys and the corresponding
probabilities as values.
:type choices: dict[object, float]
"""
if sum(choices.values()) != 1:
raise RuntimeError("The probabilities for parameter '%s'"
"do not sum up to 1" % parameter_name)
super(PChoiceParameter, self).__init__(parameter_name, choices.items())
[docs]class NormalParameter(AddedParameterDecorator):
"""
Defines a parameter as being normal distributed.
A normal distributed parameter will be sampled from
the defined distribution by the mean and standard deviation.
This parameter will be sampled from a function like:
normal(mu, sigma)
This parameter is unbound.
.. code-block:: python
>>> @NormalParameter("test", mu=0, sigma=1)
... class A(object):
... pass
"""
[docs] class Normal(object):
[docs] def __init__(self, mu, sigma):
self.mu = mu
self.sigma = sigma
[docs] def pdf(self, x):
sqrt = numpy.sqrt(2 * numpy.pi * (self.sigma ** 2))
return 1 / sqrt * numpy.e ** (-((x - self.mu) ** 2) / (2 * self.sigma ** 2))
[docs] def mean(self):
return self.mu
[docs] def __init__(self, parameter_name, mu, sigma):
"""
Create a new normal distributed parameter with `parameter_name` as name.
The mean `mu` and standard deviation `sigma` are defining the distribution
this parameter will be sampled from.
:param parameter_name: The name of the parameter to create.
:type parameter_name: str
:param mu: The mean value of the distribution for this parameter
:type mu: float
:param sigma: The standard deviation of the distribution for this parameter
:type sigma: float
"""
super(NormalParameter, self).__init__(parameter_name=parameter_name)
self.__mu = mu
self.__sigma = sigma
@property
def mu(self):
return self.__mu
@property
def sigma(self):
return self.__sigma
[docs] def plot(self):
if plt is not None:
rv = self.Normal(self.mu, self.sigma)
# mu +/- 3sigma => 99,7% confidence interval
start = self.mu - 3 * self.sigma
stop = self.mu + 3 * self.sigma
# Min 1.000 samples, max 10.000
num = min(max(stop - start * 100, 1000), 10000)
figure = plt.figure()
plt.ylabel("PDF(x)")
plt.xlabel("X")
figure.suptitle("Normal distribution for Parameter {param!s}".format(param=self.parameter_name))
x = numpy.linspace(start=start, stop=stop, num=num)
y = numpy.vectorize(lambda x_: rv.pdf(x_))(x)
axes = plt.plot(x, y, label="mu={self.mu:g}, sigma={self.sigma:g}".format(self=self))
mean_x = rv.mean()
mean_y = rv.pdf(mean_x)
y_max = 1.0 / (plt.ylim()[1] - plt.ylim()[0]) * (mean_y - plt.ylim()[0])
plt.axvline(x=mean_x, ymax=y_max, linestyle="--", color=axes[0].get_color(),
label="Mean: %g" % mean_x)
plt.legend(loc="best", fancybox=True, framealpha=0.2).draggable(True)
return figure
else:
return "{self!r}(mu={self.mu:g}, sigma={self.sigma:g})".format(self=self)
[docs]class QNormalParameter(NormalParameter, QMixin):
"""
Defines a parameter as being normal distributed but to only
take discrete values.
A q-normal distributed parameter will be sampled from the
distribution defined by a mean and a standard deviation
but it will be bound to discrete values regularized by a
regulation parameter.
Therefore the value will be sampled from a function like this:
round(normal(mu, sigma) / q) * q
This parameter is unbound.
.. code-block:: python
>>> @QNormalParameter("test", mu=0, sigma=1, q=0.5)
... class A(object):
... pass
"""
[docs] def __init__(self, parameter_name, mu, sigma, q):
"""
Creates a new q-normal distributed parameter with `parameter_name` as name.
The mean `mu` and standard deviation `sigma` are defining the distribution
this parameter will be sampled from. But it will be bound to discrete values
by the regulation parameter `q`.
:param parameter_name: The name of the parameter to create.
:type parameter_name: str
:param mu: The mean value of the distribution for the parameter
:type mu: float
:param sigma: The standard deviation of the distribution for the parameter
:type sigma: float
:param q: The regulation parameter to bind the values with.
:type q: float
"""
super(QNormalParameter, self).__init__(parameter_name=parameter_name, mu=mu, sigma=sigma)
QMixin.__init__(self, q=q)
[docs] def plot(self):
# mu +/- 3sigma => 99,7% confidence interval
if plt is not None:
rv = self.Normal(self.mu, self.sigma)
# mu +/- 3sigma => 99,7% confidence interval
start = self.round_to_q(self.mu - 3 * self.sigma)
stop = self.round_to_q(self.mu + 3 * self.sigma)
figure = plt.figure()
plt.ylabel("P(X=x)")
plt.xlabel("X")
figure.suptitle("Q-Normal distribution for Parameter {param!s}".format(param=self.parameter_name))
x = numpy.arange(start=start, stop=stop + self.q, step=self.q)
y = numpy.vectorize(lambda x_: self.calc_probability(x_, rv.pdf))(x)
axes = plt.scatter(x, y, label="mu={self.mu:g}, sigma={self.sigma:g}, q={self.q:g}".format(self=self))
mean_x = self.round_to_q(rv.mean())
mean_y = self.calc_probability(mean_x, rv.pdf)
y_max = 1.0 / (plt.ylim()[1] - plt.ylim()[0]) * (mean_y - plt.ylim()[0])
plt.axvline(x=mean_x, ymax=y_max, linestyle="--", color=axes.get_facecolor()[0],
label="Mean: %g" % mean_x)
plt.legend(loc="best", fancybox=True, framealpha=0.2).draggable(True)
return figure
else:
return "{self!r}(mu={self.mu:g}, sigma={self.sigma:g}, q={self.q:g})".format(self=self)
[docs]class LogNormalParameter(AddedParameterDecorator):
"""
Defines a parameter as being drawn from the exponential
of a normal distribution.
A log-normal distributed parameter will be sampled from
exponential of the defined distribution by a mean
and a standard deviation.
The values for this parameter will be sampled from a function like:
exp(normal(mu, sigma))
This distribution causes that the logarithm of the samples values
are being normal distributed.
This parameter is bound to positive numbers only.
.. code-block:: python
>>> @LogNormalParameter("test", shape=1, scale=1)
... class A(object):
... pass
"""
[docs] def __init__(self, parameter_name, shape, scale):
super(LogNormalParameter, self).__init__(parameter_name)
self.__shape = shape
self.__scale = scale
@property
def shape(self):
return self.__shape
@property
def scale(self):
return self.__scale
[docs] def plot(self):
if plt is not None:
rv = stats.lognorm(s=self.shape, scale=self.scale)
start, stop = rv.interval(0.99)
# Min 1.000 samples, max 10.000
num = min(max(stop - start * 100, 1000), 10000)
figure = plt.figure()
plt.ylabel("PDF(x)")
plt.xlabel("X")
figure.suptitle("Log-Normal distribution for Parameter {param!s}".format(param=self.parameter_name))
x = numpy.logspace(start=numpy.log(start), stop=numpy.log(stop), base=numpy.e, num=num)
y = rv.pdf(x)
axes = plt.plot(x, y, label="shape={self.shape:g}, scale={self.scale:g}".format(self=self))
mean_x = rv.mean()
mean_y = rv.pdf(mean_x)
y_max = 1.0 / (plt.ylim()[1] - plt.ylim()[0]) * (mean_y - plt.ylim()[0])
plt.axvline(x=mean_x, ymax=y_max, linestyle="--", color=axes[0].get_color(),
label="Mean: %g" % mean_x)
plt.legend(loc="best", fancybox=True, framealpha=0.2).draggable(True)
plt.xscale("log")
return figure
else:
return "{self!r}(shape={self.shape:g}, scale={self.scale:g})".format(self=self)
[docs]class QLogNormalParameter(LogNormalParameter, QMixin):
"""
Defines a parameter as being drawn from the exponential
of a normal distribution.
A log-normal distributed parameter will be sampled from
exponential of the defined distribution by a mean
and a standard deviation but it will be bound to discrete
values regularized by a regulation parameter.
The values for this parameter will be sampled from a function like:
round(exp(normal(mu, sigma)) / q) * q
This distribution causes that the logarithm of the samples values
are being normal distributed.
This parameter is bound to positive number only.
.. code-block:: python
>>> @QLogNormalParameter("test", shape=1, scale=1, q=0.5)
... class A(object):
... pass
"""
[docs] def __init__(self, parameter_name, shape, scale, q):
LogNormalParameter.__init__(self, parameter_name, shape, scale)
QMixin.__init__(self, q)
[docs] def plot(self):
if plt is not None:
rv = stats.lognorm(s=self.shape, scale=self.scale)
start, stop = rv.interval(0.99)
figure = plt.figure()
plt.ylabel("P(X=x)")
plt.xlabel("X")
figure.suptitle("Q-Log-Normal distribution for Parameter {param!s}".format(param=self.parameter_name))
x = numpy.arange(start=self.round_to_q(start), stop=self.round_to_q(stop) + self.q, step=self.q)
y = numpy.vectorize(lambda x_: self.calc_probability(x_, rv.pdf))(x)
axes = plt.scatter(x, y,
label="shape={self.shape:g}, scale={self.scale:g}, q={self.q:g}".format(self=self))
mean_x = self.round_to_q(rv.mean())
mean_y = self.calc_probability(mean_x, rv.pdf)
y_max = 1.0 / (plt.ylim()[1] - plt.ylim()[0]) * (mean_y - plt.ylim()[0])
plt.axvline(x=mean_x, ymax=y_max, linestyle="--", color=axes.get_facecolor()[0],
label="Mean: %g" % mean_x)
plt.legend(loc="best", fancybox=True, framealpha=0.2).draggable(True)
plt.xscale("log")
return figure
else:
return "{self!r}(shape={self.shape:g}, scale={self.scale:g}, q={self.q:g})".format(self=self)
[docs]class NoOptimizationParameter(AddedParameterDecorator):
"""
Defines a previously defined parameter as not being
an optimization parameter at all.
This decorator can be used in derived classes where
optimization parameters of base classes are given concrete values
or don't matter at all.
.. code-block:: python
>>> @NoOptimizationParameter("test")
... class A(object):
... pass
"""
[docs] def plot(self):
return repr(self)
PARAMETER_TYPES = {
"Choice": ChoiceParameter,
"Boolean": BooleanParameter,
"PChoice": PChoiceParameter,
"Normal": NormalParameter,
"Uniform": UniformParameter,
"QNormal": QNormalParameter,
"QUniform": QUniformParameter,
"LogNormal": LogNormalParameter,
"LogUniform": LogUniformParameter,
"QLogNormal": QLogNormalParameter,
"QLogUniform": QLogUniformParameter,
"NoOptimization": NoOptimizationParameter
}
