# Copyright 2016 The TensorFlow Authors. All Rights Reserved.

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

# http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

# ==============================================================================

"""Implementation of tf.metrics module."""

from __future__ import absolute_import

from __future__ import division

from __future__ import print_function

from tensorflow.python.framework import dtypes

from tensorflow.python.framework import ops

from tensorflow.python.framework import sparse_tensor

from tensorflow.python.ops import array_ops

from tensorflow.python.ops import check_ops

from tensorflow.python.ops import confusion_matrix

from tensorflow.python.ops import control_flow_ops

from tensorflow.python.ops import math_ops

from tensorflow.python.ops import nn

from tensorflow.python.ops import sets

from tensorflow.python.ops import sparse_ops

from tensorflow.python.ops import state_ops

from tensorflow.python.ops import variable_scope

from tensorflow.python.ops import variables

from tensorflow.python.ops import weights_broadcast_ops

def _local_variable(initial_value, validate_shape=True, name=None):

"""Create variable and add it to `GraphKeys.LOCAL_VARIABLES` collection.

Args:

initial_value: See variables.Variable.__init__.

validate_shape: See variables.Variable.__init__.

name: See variables.Variable.__init__.

Returns:

New variable.

"""

return variables.Variable(

initial_value, trainable=False,

collections=[ops.GraphKeys.LOCAL_VARIABLES],

validate_shape=validate_shape, name=name)

def _remove_squeezable_dimensions(predictions, labels, weights):

"""Internal version of `remove_squeezable_dimensions` which handles weights.

Squeezes `predictions` and `labels` if their rank differs by 1.

Squeezes `weights` if its rank is 1 more than the new rank of `predictions`

This will use static shape if available. Otherwise, it will add graph

operations, which could result in a performance hit.

Args:

predictions: Predicted values, a `Tensor` of arbitrary dimensions.

labels: Optional label `Tensor` whose dimensions match `predictions`.

weights: Optional weight `Tensor`. It will be squeezed if its rank is 1

more than the new rank of `predictions`

Returns:

Tuple of `predictions`, `labels` and `weights`, possibly with the last

dimension squeezed.

"""

predictions = ops.convert_to_tensor(predictions)

if labels is not None:

labels, predictions = confusion_matrix.remove_squeezable_dimensions(

labels, predictions)

predictions.get_shape().assert_is_compatible_with(labels.get_shape())

if weights is not None:

weights = ops.convert_to_tensor(weights)

predictions_shape = predictions.get_shape()

predictions_rank = predictions_shape.ndims

weights_shape = weights.get_shape()

weights_rank = weights_shape.ndims

# TODO(ptucker): Add logic to handle weights rank 1 less than predictions &

# labels.

if (predictions_rank is not None) and (weights_rank is not None):

# Use static rank.

if weights_rank - predictions_rank == 1:

weights = array_ops.squeeze(weights, [-1])

elif ((weights_rank is None) or

((weights_rank > 0) and

weights_shape.dims[-1].is_compatible_with(1))):

# Use dynamic rank.

weights = control_flow_ops.cond(

math_ops.equal(array_ops.rank(weights),

math_ops.add(array_ops.rank(predictions), 1)),

lambda: array_ops.squeeze(weights, [-1]),

lambda: weights)

return predictions, labels, weights

def _maybe_expand_labels(labels, predictions):

"""If necessary, expand `labels` along last dimension to match `predictions`.

Args:

labels: `Tensor` or `SparseTensor` with shape

[D1, ... DN, num_labels] or [D1, ... DN]. The latter implies

num_labels=1, in which case the result is an expanded `labels` with shape

[D1, ... DN, 1].

predictions: `Tensor` with shape [D1, ... DN, num_classes].

Returns:

`labels` with the same rank as `predictions`.

Raises:

ValueError: if `labels` has invalid shape.

"""

with ops.name_scope(None, 'expand_labels', (labels, predictions)) as scope:

labels = sparse_tensor.convert_to_tensor_or_sparse_tensor(labels)

# If sparse, expand sparse shape.

if isinstance(labels, sparse_tensor.SparseTensor):

return control_flow_ops.cond(

math_ops.equal(

array_ops.rank(predictions),

array_ops.size(labels.dense_shape) + 1),

lambda: sparse_ops.sparse_reshape( # pylint: disable=g-long-lambda

labels,

shape=array_ops.concat((labels.dense_shape, (1,)), 0),

name=scope),

lambda: labels)

# Otherwise, try to use static shape.

labels_rank = labels.get_shape().ndims

if labels_rank is not None:

predictions_rank = predictions.get_shape().ndims

if predictions_rank is not None:

if predictions_rank == labels_rank:

return labels

if predictions_rank == labels_rank + 1:

return array_ops.expand_dims(labels, -1, name=scope)

raise ValueError(

'Unexpected labels shape %s for predictions shape %s.' % (

labels.get_shape(), predictions.get_shape()))

# Otherwise, use dynamic shape.

return control_flow_ops.cond(

math_ops.equal(array_ops.rank(predictions), array_ops.rank(labels) + 1),

lambda: array_ops.expand_dims(labels, -1, name=scope),

lambda: labels)

def _create_local(name, shape, collections=None, validate_shape=True,

dtype=dtypes.float32):

"""Creates a new local variable.

Args:

name: The name of the new or existing variable.

shape: Shape of the new or existing variable.

collections: A list of collection names to which the Variable will be added.

validate_shape: Whether to validate the shape of the variable.

dtype: Data type of the variables.

Returns:

The created variable.

"""

# Make sure local variables are added to tf.GraphKeys.LOCAL_VARIABLES

collections = list(collections or [])

collections += [ops.GraphKeys.LOCAL_VARIABLES]

return variables.Variable(

initial_value=array_ops.zeros(shape, dtype=dtype),

name=name,

trainable=False,

collections=collections,

validate_shape=validate_shape)

def _safe_div(numerator, denominator, name):

"""Divides two values, returning 0 if the denominator is <= 0.

Args:

numerator: A real `Tensor`.

denominator: A real `Tensor`, with dtype matching `numerator`.

name: Name for the returned op.

Returns:

0 if `denominator` <= 0, else `numerator` / `denominator`

"""

return array_ops.where(

math_ops.greater(denominator, 0),

math_ops.truediv(numerator, denominator),

0,

name=name)

def _safe_scalar_div(numerator, denominator, name):

"""Divides two values, returning 0 if the denominator is 0.

Args:

numerator: A scalar `float64` `Tensor`.

denominator: A scalar `float64` `Tensor`.

name: Name for the returned op.

Returns:

0 if `denominator` == 0, else `numerator` / `denominator`

"""

numerator.get_shape().with_rank_at_most(1)

denominator.get_shape().with_rank_at_most(1)

return control_flow_ops.cond(

math_ops.equal(

array_ops.constant(0.0, dtype=dtypes.float64), denominator),

lambda: array_ops.constant(0.0, dtype=dtypes.float64),

lambda: math_ops.div(numerator, denominator),

name=name)

def _streaming_confusion_matrix(labels, predictions, num_classes, weights=None):

"""Calculate a streaming confusion matrix.

Calculates a confusion matrix. For estimation over a stream of data,

the function creates an `update_op` operation.

Args:

labels: A `Tensor` of ground truth labels with shape [batch size] and of

type `int32` or `int64`. The tensor will be flattened if its rank > 1.

predictions: A `Tensor` of prediction results for semantic labels, whose

shape is [batch size] and type `int32` or `int64`. The tensor will be

flattened if its rank > 1.

num_classes: The possible number of labels the prediction task can

have. This value must be provided, since a confusion matrix of

dimension = [num_classes, num_classes] will be allocated.

weights: Optional `Tensor` whose rank is either 0, or the same rank as

`labels`, and must be broadcastable to `labels` (i.e., all dimensions must

be either `1`, or the same as the corresponding `labels` dimension).

Returns:

total_cm: A `Tensor` representing the confusion matrix.

update_op: An operation that increments the confusion matrix.

"""

# Local variable to accumulate the predictions in the confusion matrix.

cm_dtype = dtypes.int64 if weights is not None else dtypes.float64

total_cm = _create_local(

'total_confusion_matrix',

shape=[num_classes, num_classes],

dtype=cm_dtype)

# Cast the type to int64 required by confusion_matrix_ops.

predictions = math_ops.to_int64(predictions)

labels = math_ops.to_int64(labels)

num_classes = math_ops.to_int64(num_classes)

# Flatten the input if its rank > 1.

if predictions.get_shape().ndims > 1:

predictions = array_ops.reshape(predictions, [-1])

if labels.get_shape().ndims > 1:

labels = array_ops.reshape(labels, [-1])

if (weights is not None) and (weights.get_shape().ndims > 1):

weights = array_ops.reshape(weights, [-1])

# Accumulate the prediction to current confusion matrix.

current_cm = confusion_matrix.confusion_matrix(

labels, predictions, num_classes, weights=weights, dtype=cm_dtype)

update_op = state_ops.assign_add(total_cm, current_cm)

return total_cm, update_op

def mean(values, weights=None, metrics_collections=None,

updates_collections=None, name=None):

"""Computes the (weighted) mean of the given values.

The `mean` function creates two local variables, `total` and `count`

that are used to compute the average of `values`. This average is ultimately

returned as `mean` which is an idempotent operation that simply divides

`total` by `count`.

For estimation of the metric over a stream of data, the function creates an

`update_op` operation that updates these variables and returns the `mean`.

`update_op` increments `total` with the reduced sum of the product of `values`

and `weights`, and it increments `count` with the reduced sum of `weights`.

If `weights` is `None`, weights default to 1. Use weights of 0 to mask values.

Args:

values: A `Tensor` of arbitrary dimensions.

weights: Optional `Tensor` whose rank is either 0, or the same rank as

`values`, and must be broadcastable to `values` (i.e., all dimensions must

be either `1`, or the same as the corresponding `values` dimension).

metrics_collections: An optional list of collections that `mean`

should be added to.

updates_collections: An optional list of collections that `update_op`

should be added to.

name: An optional variable_scope name.

Returns:

mean: A `Tensor` representing the current mean, the value of `total` divided

by `count`.

update_op: An operation that increments the `total` and `count` variables

appropriately and whose value matches `mean_value`.

Raises:

ValueError: If `weights` is not `None` and its shape doesn't match `values`,

or if either `metrics_collections` or `updates_collections` are not a list

or tuple.

"""

with variable_scope.variable_scope(name, 'mean', (values, weights)):

values = math_ops.to_float(values)

total = _create_local('total', shape=[])

count = _create_local('count', shape=[])

if weights is None:

num_values = math_ops.to_float(array_ops.size(values))

else:

values, _, weights = _remove_squeezable_dimensions(

predictions=values, labels=None, weights=weights)

weights = weights_broadcast_ops.broadcast_weights(

math_ops.to_float(weights), values)

values = math_ops.multiply(values, weights)

num_values = math_ops.reduce_sum(weights)

update_total_op = state_ops.assign_add(total, math_ops.reduce_sum(values))

with ops.control_dependencies([values]):

update_count_op = state_ops.assign_add(count, num_values)

mean_t = _safe_div(total, count, 'value')

update_op = _safe_div(update_total_op, update_count_op, 'update_op')

if metrics_collections:

ops.add_to_collections(metrics_collections, mean_t)

if updates_collections:

ops.add_to_collections(updates_collections, update_op)

return mean_t, update_op

def accuracy(labels, predictions, weights=None, metrics_collections=None,

updates_collections=None, name=None):

"""Calculates how often `predictions` matches `labels`.

The `accuracy` function creates two local variables, `total` and

`count` that are used to compute the frequency with which `predictions`

matches `labels`. This frequency is ultimately returned as `accuracy`: an

idempotent operation that simply divides `total` by `count`.

For estimation of the metric over a stream of data, the function creates an

`update_op` operation that updates these variables and returns the `accuracy`.

Internally, an `is_correct` operation computes a `Tensor` with elements 1.0

where the corresponding elements of `predictions` and `labels` match and 0.0

otherwise. Then `update_op` increments `total` with the reduced sum of the

product of `weights` and `is_correct`, and it increments `count` with the

reduced sum of `weights`.

If `weights` is `None`, weights default to 1. Use weights of 0 to mask values.

Args:

labels: The ground truth values, a `Tensor` whose shape matches

`predictions`.

predictions: The predicted values, a `Tensor` of any shape.

weights: Optional `Tensor` whose rank is either 0, or the same rank as

`labels`, and must be broadcastable to `labels` (i.e., all dimensions must

be either `1`, or the same as the corresponding `labels` dimension).

metrics_collections: An optional list of collections that `accuracy` should

be added to.

updates_collections: An optional list of collections that `update_op` should

be added to.

name: An optional variable_scope name.

Returns:

accuracy: A `Tensor` representing the accuracy, the value of `total` divided

by `count`.

update_op: An operation that increments the `total` and `count` variables

appropriately and whose value matches `accuracy`.

Raises:

ValueError: If `predictions` and `labels` have mismatched shapes, or if

`weights` is not `None` and its shape doesn't match `predictions`, or if

either `metrics_collections` or `updates_collections` are not a list or

tuple.

"""

predictions, labels, weights = _remove_squeezable_dimensions(

predictions=predictions, labels=labels, weights=weights)

predictions.get_shape().assert_is_compatible_with(labels.get_shape())

if labels.dtype != predictions.dtype:

predictions = math_ops.cast(predictions, labels.dtype)

is_correct = math_ops.to_float(math_ops.equal(predictions, labels))

return mean(is_correct, weights, metrics_collections,

updates_collections, name or 'accuracy')

def _confusion_matrix_at_thresholds(

labels, predictions, thresholds, weights=None, includes=None):

"""Computes true_positives, false_negatives, true_negatives, false_positives.

This function creates up to four local variables, `true_positives`,

`true_negatives`, `false_positives` and `false_negatives`.

`true_positive[i]` is defined as the total weight of values in `predictions`

above `thresholds[i]` whose corresponding entry in `labels` is `True`.

`false_negatives[i]` is defined as the total weight of values in `predictions`

at most `thresholds[i]` whose corresponding entry in `labels` is `True`.

`true_negatives[i]` is defined as the total weight of values in `predictions`

at most `thresholds[i]` whose corresponding entry in `labels` is `False`.

`false_positives[i]` is defined as the total weight of values in `predictions`

above `thresholds[i]` whose corresponding entry in `labels` is `False`.

For estimation of these metrics over a stream of data, for each metric the

function respectively creates an `update_op` operation that updates the

variable and returns its value.

If `weights` is `None`, weights default to 1. Use weights of 0 to mask values.

Args:

labels: A `Tensor` whose shape matches `predictions`. Will be cast to

`bool`.

predictions: A floating point `Tensor` of arbitrary shape and whose values

are in the range `[0, 1]`.

thresholds: A python list or tuple of float thresholds in `[0, 1]`.

weights: Optional `Tensor` whose rank is either 0, or the same rank as

`labels`, and must be broadcastable to `labels` (i.e., all dimensions must

be either `1`, or the same as the corresponding `labels` dimension).

includes: Tuple of keys to return, from 'tp', 'fn', 'tn', fp'. If `None`,

default to all four.

Returns:

values: Dict of variables of shape `[len(thresholds)]`. Keys are from

`includes`.

update_ops: Dict of operations that increments the `values`. Keys are from

`includes`.

Raises:

ValueError: If `predictions` and `labels` have mismatched shapes, or if

`weights` is not `None` and its shape doesn't match `predictions`, or if

`includes` contains invalid keys.

"""

all_includes = ('tp', 'fn', 'tn', 'fp')

if includes is None:

includes = all_includes

else:

for include in includes:

if include not in all_includes:

raise ValueError('Invaild key: %s.' % include)

predictions, labels, weights = _remove_squeezable_dimensions(

predictions=math_ops.to_float(predictions),

labels=math_ops.cast(labels, dtype=dtypes.bool),

weights=weights)

num_thresholds = len(thresholds)

# Reshape predictions and labels.

predictions_2d = array_ops.reshape(predictions, [-1, 1])

labels_2d = array_ops.reshape(

math_ops.cast(labels, dtype=dtypes.bool), [1, -1])

# Use static shape if known.

num_predictions = predictions_2d.get_shape().as_list()[0]

# Otherwise use dynamic shape.

if num_predictions is None:

num_predictions = array_ops.shape(predictions_2d)[0]

thresh_tiled = array_ops.tile(

array_ops.expand_dims(array_ops.constant(thresholds), [1]),

array_ops.stack([1, num_predictions]))

# Tile the predictions after thresholding them across different thresholds.

pred_is_pos = math_ops.greater(

array_ops.tile(array_ops.transpose(predictions_2d), [num_thresholds, 1]),

thresh_tiled)

if ('fn' in includes) or ('tn' in includes):

pred_is_neg = math_ops.logical_not(pred_is_pos)

# Tile labels by number of thresholds

label_is_pos = array_ops.tile(labels_2d, [num_thresholds, 1])

if ('fp' in includes) or ('tn' in includes):

label_is_neg = math_ops.logical_not(label_is_pos)

if weights is not None:

weights = weights_broadcast_ops.broadcast_weights(

math_ops.to_float(weights), predictions)

weights_tiled = array_ops.tile(array_ops.reshape(

weights, [1, -1]), [num_thresholds, 1])

thresh_tiled.get_shape().assert_is_compatible_with(

weights_tiled.get_shape())

else:

weights_tiled = None

values = {}

update_ops = {}

if 'tp' in includes:

true_p = _create_local('true_positives', shape=[num_thresholds])

is_true_positive = math_ops.to_float(

math_ops.logical_and(label_is_pos, pred_is_pos))

if weights_tiled is not None:

is_true_positive *= weights_tiled

update_ops['tp'] = state_ops.assign_add(

true_p, math_ops.reduce_sum(is_true_positive, 1))

values['tp'] = true_p

if 'fn' in includes:

false_n = _create_local('false_negatives', shape=[num_thresholds])

is_false_negative = math_ops.to_float(

math_ops.logical_and(label_is_pos, pred_is_neg))

if weights_tiled is not None:

is_false_negative *= weights_tiled

update_ops['fn'] = state_ops.assign_add(

false_n, math_ops.reduce_sum(is_false_negative, 1))

values['fn'] = false_n

if 'tn' in includes:

true_n = _create_local('true_negatives', shape=[num_thresholds])

is_true_negative = math_ops.to_float(

math_ops.logical_and(label_is_neg, pred_is_neg))

if weights_tiled is not None:

is_true_negative *= weights_tiled

update_ops['tn'] = state_ops.assign_add(

true_n, math_ops.reduce_sum(is_true_negative, 1))

values['tn'] = true_n

if 'fp' in includes:

false_p = _create_local('false_positives', shape=[num_thresholds])

is_false_positive = math_ops.to_float(

math_ops.logical_and(label_is_neg, pred_is_pos))

if weights_tiled is not None:

is_false_positive *= weights_tiled

update_ops['fp'] = state_ops.assign_add(

false_p, math_ops.reduce_sum(is_false_positive, 1))

values['fp'] = false_p

return values, update_ops

def auc(labels, predictions, weights=None, num_thresholds=200,

metrics_collections=None, updates_collections=None,

curve='ROC', name=None):

"""Computes the approximate AUC via a Riemann sum.

The `auc` function creates four local variables, `true_positives`,

`true_negatives`, `false_positives` and `false_negatives` that are used to

compute the AUC. To discretize the AUC curve, a linearly spaced set of

thresholds is used to compute pairs of recall and precision values. The area

under the ROC-curve is therefore computed using the height of the recall

values by the false positive rate, while the area under the PR-curve is the

computed using the height of the precision values by the recall.

This value is ultimately returned as `auc`, an idempotent operation that

computes the area under a discretized curve of precision versus recall values

(computed using the aforementioned variables). The `num_thresholds` variable

controls the degree of discretization with larger numbers of thresholds more

closely approximating the true AUC. The quality of the approximation may vary

dramatically depending on `num_thresholds`.

For best results, `predictions` should be distributed approximately uniformly

in the range [0, 1] and not peaked around 0 or 1. The quality of the AUC

approximation may be poor if this is not the case.

For estimation of the metric over a stream of data, the function creates an

`update_op` operation that updates these variables and returns the `auc`.

If `weights` is `None`, weights default to 1. Use weights of 0 to mask values.

Args:

labels: A `Tensor` whose shape matches `predictions`. Will be cast to

`bool`.

predictions: A floating point `Tensor` of arbitrary shape and whose values

are in the range `[0, 1]`.

weights: Optional `Tensor` whose rank is either 0, or the same rank as

`labels`, and must be broadcastable to `labels` (i.e., all dimensions must

be either `1`, or the same as the corresponding `labels` dimension).

num_thresholds: The number of thresholds to use when discretizing the roc

curve.

metrics_collections: An optional list of collections that `auc` should be

added to.

updates_collections: An optional list of collections that `update_op` should

be added to.

curve: Specifies the name of the curve to be computed, 'ROC' [default] or

'PR' for the Precision-Recall-curve.

name: An optional variable_scope name.

Returns:

auc: A scalar `Tensor` representing the current area-under-curve.

update_op: An operation that increments the `true_positives`,

`true_negatives`, `false_positives` and `false_negatives` variables

appropriately and whose value matches `auc`.

Raises:

ValueError: If `predictions` and `labels` have mismatched shapes, or if

`weights` is not `None` and its shape doesn't match `predictions`, or if

either `metrics_collections` or `updates_collections` are not a list or

tuple.

"""

with variable_scope.variable_scope(

name, 'auc', (labels, predictions, weights)):

if curve != 'ROC' and curve != 'PR':

raise ValueError('curve must be either ROC or PR, %s unknown' %

(curve))

kepsilon = 1e-7 # to account for floating point imprecisions

thresholds = [(i + 1) * 1.0 / (num_thresholds - 1)

for i in range(num_thresholds-2)]

thresholds = [0.0 - kepsilon] + thresholds + [1.0 + kepsilon]

values, update_ops = _confusion_matrix_at_thresholds(

labels, predictions, thresholds, weights)

# Add epsilons to avoid dividing by 0.

epsilon = 1.0e-6

def compute_auc(tp, fn, tn, fp, name):

"""Computes the roc-auc or pr-auc based on confusion counts."""

rec = math_ops.div(tp + epsilon, tp + fn + epsilon)

if curve == 'ROC':

fp_rate = math_ops.div(fp, fp + tn + epsilon)

x = fp_rate

y = rec

else: # curve == 'PR'.

prec = math_ops.div(tp + epsilon, tp + fp + epsilon)

x = rec

y = prec

return math_ops.reduce_sum(math_ops.multiply(

x[:num_thresholds - 1] - x[1:],

(y[:num_thresholds - 1] + y[1:]) / 2.), name=name)

# sum up the areas of all the trapeziums

auc_value = compute_auc(

values['tp'], values['fn'], values['tn'], values['fp'], 'value')

update_op = compute_auc(

update_ops['tp'], update_ops['fn'], update_ops['tn'], update_ops['fp'],

'update_op')

if metrics_collections:

ops.add_to_collections(metrics_collections, auc_value)

if updates_collections:

ops.add_to_collections(updates_collections, update_op)

return auc_value, update_op

def mean_absolute_error(labels, predictions, weights=None,

metrics_collections=None,

updates_collections=None,

name=None):

"""Computes the mean absolute error between the labels and predictions.

The `mean_absolute_error` function creates two local variables,

`total` and `count` that are used to compute the mean absolute error. This

average is weighted by `weights`, and it is ultimately returned as

`mean_absolute_error`: an idempotent operation that simply divides `total` by

`count`.

For estimation of the metric over a stream of data, the function creates an

`update_op` operation that updates these variables and returns the

`mean_absolute_error`. Internally, an `absolute_errors` operation computes the

absolute value of the differences between `predictions` and `labels`. Then

`update_op` increments `total` with the reduced sum of the product of

`weights` and `absolute_errors`, and it increments `count` with the reduced

sum of `weights`

If `weights` is `None`, weights default to 1. Use weights of 0 to mask values.

Args:

labels: A `Tensor` of the same shape as `predictions`.

predictions: A `Tensor` of arbitrary shape.

weights: Optional `Tensor` whose rank is either 0, or the same rank as

`labels`, and must be broadcastable to `labels` (i.e., all dimensions must

be either `1`, or the same as the corresponding `labels` dimension).

metrics_collections: An optional list of collections that

`mean_absolute_error` should be added to.

updates_collections: An optional list of collections that `update_op` should

be added to.

name: An optional variable_scope name.

Returns:

mean_absolute_error: A `Tensor` representing the current mean, the value of

`total` divided by `count`.

update_op: An operation that increments the `total` and `count` variables

appropriately and whose value matches `mean_absolute_error`.

Raises:

ValueError: If `predictions` and `labels` have mismatched shapes, or if

`weights` is not `None` and its shape doesn't match `predictions`, or if

either `metrics_collections` or `updates_collections` are not a list or

tuple.

"""

predictions, labels, weights = _remove_squeezable_dimensions(

predictions=predictions, labels=labels, weights=weights)

absolute_errors = math_ops.abs(predictions - labels)

return mean(absolute_errors, weights, metrics_collections,

updates_collections, name or 'mean_absolute_error')

def mean_cosine_distance(labels, predictions, dim, weights=None,

metrics_collections=None,

updates_collections=None,

name=None):

"""Computes the cosine distance between the labels and predictions.

The `mean_cosine_distance` function creates two local variables,

`total` and `count` that are used to compute the average cosine distance

between `predictions` and `labels`. This average is weighted by `weights`,

and it is ultimately returned as `mean_distance`, which is an idempotent

operation that simply divides `total` by `count`.

For estimation of the metric over a stream of data, the function creates an

`update_op` operation that updates these variables and returns the

`mean_distance`.

If `weights` is `None`, weights default to 1. Use weights of 0 to mask values.

Args:

labels: A `Tensor` of arbitrary shape.

predictions: A `Tensor` of the same shape as `labels`.

dim: The dimension along which the cosine distance is computed.

weights: Optional `Tensor` whose rank is either 0, or the same rank as

`labels`, and must be broadcastable to `labels` (i.e., all dimensions must

be either `1`, or the same as the corresponding `labels` dimension). Also,

dimension `dim` must be `1`.

metrics_collections: An optional list of collections that the metric

value variable should be added to.

updates_collections: An optional list of collections that the metric update

ops should be added to.

name: An optional variable_scope name.

Returns:

mean_distance: A `Tensor` representing the current mean, the value of

`total` divided by `count`.

update_op: An operation that increments the `total` and `count` variables

appropriately.

Raises:

ValueError: If `predictions` and `labels` have mismatched shapes, or if

`weights` is not `None` and its shape doesn't match `predictions`, or if

either `metrics_collections` or `updates_collections` are not a list or

tuple.

"""

predictions, labels, weights = _remove_squeezable_dimensions(

predictions=predictions, labels=labels, weights=weights)

radial_diffs = math_ops.multiply(predictions, labels)

radial_diffs = math_ops.reduce_sum(radial_diffs,

reduction_indices=[dim,],

keep_dims=True)

mean_distance, update_op = mean(radial_diffs, weights,

None,

None,

name or 'mean_cosine_distance')

mean_distance = math_ops.subtract(1.0, mean_distance)

update_op = math_ops.subtract(1.0, update_op)

if metrics_collections:

ops.add_to_collections(metrics_collections, mean_distance)

if updates_collections:

ops.add_to_collections(updates_collections, update_op)

return mean_distance, update_op

def mean_per_class_accuracy(labels,

predictions,

num_classes,

weights=None,

metrics_collections=None,

updates_collections=None,

name=None):

"""Calculates the mean of the per-class accuracies.

Calculates the accuracy for each class, then takes the mean of that.

For estimation of the metric over a stream of data, the function creates an

`update_op` operation that updates these variables and returns the

`mean_accuracy`.

If `weights` is `None`, weights default to 1. Use weights of 0 to mask values.

Args:

labels: A `Tensor` of ground truth labels with shape [batch size] and of

type `int32` or `int64`. The tensor will be flattened if its rank > 1.

predictions: A `Tensor` of prediction results for semantic labels, whose

shape is [batch size] and type `int32` or `int64`. The tensor will be

flattened if its rank > 1.

num_classes: The possible number of labels the prediction task can

have. This value must be provided, since a confusion matrix of

dimension = [num_classes, num_classes] will be allocated.

weights: Optional `Tensor` whose rank is either 0, or the same rank as

`labels`, and must be broadcastable to `labels` (i.e., all dimensions must

be either `1`, or the same as the corresponding `labels` dimension).

metrics_collections: An optional list of collections that

`mean_per_class_accuracy'

should be added to.

updates_collections: An optional list of collections `update_op` should be

added to.

name: An optional variable_scope name.

Returns:

mean_accuracy: A `Tensor` representing the mean per class accuracy.

update_op: An operation that increments the confusion matrix.

Raises:

ValueError: If `predictions` and `labels` have mismatched shapes, or if

`weights` is not `None` and its shape doesn't match `predictions`, or if

either `metrics_collections` or `updates_collections` are not a list or

tuple.

"""

with variable_scope.variable_scope(name, 'mean_accuracy',

(predictions, labels, weights)):

# Check if shape is compatible.

predictions.get_shape().assert_is_compatible_with(labels.get_shape())

total_cm, update_op = _streaming_confusion_matrix(

labels, predictions, num_classes, weights=weights)

def compute_mean_accuracy(name):

"""Compute the mean per class accuracy via the confusion matrix."""

per_row_sum = math_ops.to_float(math_ops.reduce_sum(total_cm, 1))

cm_diag = math_ops.to_float(array_ops.diag_part(total_cm))

denominator = per_row_sum

# If the value of the denominator is 0, set it to 1 to avoid

# zero division.

denominator = array_ops.where(

math_ops.greater(denominator, 0), denominator,

array_ops.ones_like(denominator))

accuracies = math_ops.div(cm_diag, denominator)

return math_ops.reduce_mean(accuracies, name=name)

mean_accuracy_v = compute_mean_accuracy('mean_accuracy')

if metrics_collections:

ops.add_to_collections(metrics_collections, mean_accuracy_v)

if updates_collections:

ops.add_to_collections(updates_collections, update_op)

return mean_accuracy_v, update_op

def mean_iou(labels,

predictions,

num_classes,

weights=None,

metrics_collections=None,

updates_collections=None,

name=None):

"""Calculate per-step mean Intersection-Over-Union (mIOU).

Mean Intersection-Over-Union is a common evaluation metric for

semantic image segmentation, which first computes the IOU for each

semantic class and then computes the average over classes.

IOU is defined as follows:

IOU = true_positive / (true_positive + false_positive + false_negative).

The predictions are accumulated in a confusion matrix, weighted by `weights`,

and mIOU is then calculated from it.

For estimation of the metric over a stream of data, the function creates an

`update_op` operation that updates these variables and returns the `mean_iou`.

If `weights` is `None`, weights default to 1. Use weights of 0 to mask values.

Args:

labels: A `Tensor` of ground truth labels with shape [batch size] and of

type `int32` or `int64`. The tensor will be flattened if its rank > 1.

predictions: A `Tensor` of prediction results for semantic labels, whose

shape is [batch size] and type `int32` or `int64`. The tensor will be

flattened if its rank > 1.

num_classes: The possible number of labels the prediction task can

have. This value must be provided, since a confusion matrix of

dimension = [num_classes, num_classes] will be allocated.

weights: Optional `Tensor` whose rank is either 0, or the same rank as

`labels`, and must be broadcastable to `labels` (i.e., all dimensions must

be either `1`, or the same as the corresponding `labels` dimension).

metrics_collections: An optional list of collections that `mean_iou`

should be added to.

updates_collections: An optional list of collections `update_op` should be

added to.

name: An optional variable_scope name.

Returns:

mean_iou: A `Tensor` representing the mean intersection-over-union.

update_op: An operation that increments the confusion matrix.

Raises:

ValueError: If `predictions` and `labels` have mismatched shapes, or if

`weights` is not `None` and its shape doesn't match `predictions`, or if

either `metrics_collections` or `updates_collections` are not a list or

tuple.

"""

with variable_scope.variable_scope(

name, 'mean_iou', (predictions, labels, weights)):

# Check if shape is compatible.

predictions.get_shape().assert_is_compatible_with(labels.get_shape())

total_cm, update_op = _streaming_confusion_matrix(labels, predictions,

num_classes, weights)

def compute_mean_iou(name):

"""Compute the mean intersection-over-union via the confusion matrix."""

sum_over_row = math_ops.to_float(math_ops.reduce_sum(total_cm, 0))

sum_over_col = math_ops.to_float(math_ops.reduce_sum(total_cm, 1))

cm_diag = math_ops.to_float(array_ops.diag_part(total_cm))

denominator = sum_over_row + sum_over_col - cm_diag

# If the value of the denominator is 0, set it to 1 to avoid

# zero division.

denominator = array_ops.where(

math_ops.greater(denominator, 0),

denominator,

array_ops.ones_like(denominator))

iou = math_ops.div(cm_diag, denominator)

return math_ops.reduce_mean(iou, name=name)

mean_iou_v = compute_mean_iou('mean_iou')

if metrics_collections:

ops.add_to_collections(metrics_collections, mean_iou_v)

if updates_collections:

ops.add_to_collections(updates_collections, update_op)

return mean_iou_v, update_op

def mean_relative_error(labels, predictions, normalizer, weights=None,

metrics_collections=None,

updates_collections=None,

name=None):

"""Computes the mean relative error by normalizing with the given values.

The `mean_relative_error` function creates two local variables,

`total` and `count` that are used to compute the mean relative absolute error.

This average is weighted by `weights`, and it is ultimately returned as

`mean_relative_error`: an idempotent operation that simply divides `total` by

`count`.

For estimation of the metric over a stream of data, the function creates an

`update_op` operation that updates these variables and returns the

`mean_reative_error`. Internally, a `relative_errors` operation divides the

absolute value of the differences between `predictions` and `labels` by the

`normalizer`. Then `update_op` increments `total` with the reduced sum of the

product of `weights` and `relative_errors`, and it increments `count` with the

reduced sum of `weights`.

If `weights` is `None`, weights default to 1. Use weights of 0 to mask values.

Args:

labels: A `Tensor` of the same shape as `predictions`.

predictions: A `Tensor` of arbitrary shape.

normalizer: A `Tensor` of the same shape as `predictions`.

weights: Optional `Tensor` whose rank is either 0, or the same rank as

`labels`, and must be broadcastable to `labels` (i.e., all dimensions must

be either `1`, or the same as the corresponding `labels` dimension).

metrics_collections: An optional list of collections that

`mean_relative_error` should be added to.

updates_collections: An optional list of collections that `update_op` should

be added to.

name: An optional variable_scope name.

Returns:

mean_relative_error: A `Tensor` representing the current mean, the value of

`total` divided by `count`.

update_op: An operation that increments the `total` and `count` variables

appropriately and whose value matches `mean_relative_error`.

Raises:

ValueError: If `predictions` and `labels` have mismatched shapes, or if

`weights` is not `None` and its shape doesn't match `predictions`, or if

either `metrics_collections` or `updates_collections` are not a list or

tuple.

"""

predictions, labels, weights = _remove_squeezable_dimensions(

predictions=predictions, labels=labels, weights=weights)

predictions, normalizer = confusion_matrix.remove_squeezable_dimensions(

predictions, normalizer)

predictions.get_shape().assert_is_compatible_with(normalizer.get_shape())

relative_errors = array_ops.where(

math_ops.equal(normalizer, 0.0),

array_ops.zeros_like(labels),

math_ops.div(math_ops.abs(labels - predictions), normalizer))

return mean(relative_errors, weights, metrics_collections,

updates_collections, name or 'mean_relative_error')

def mean_squared_error(labels, predictions, weights=None,

metrics_collections=None,

updates_collections=None,

name=None):

"""Computes the mean squared error between the labels and predictions.

The `mean_squared_error` function creates two local variables,

`total` and `count` that are used to compute the mean squared error.

This average is weighted by `weights`, and it is ultimately returned as

`mean_squared_error`: an idempotent operation that simply divides `total` by

`count`.

For estimation of the metric over a stream of data, the function creates an

`update_op` operation that updates these variables and returns the

`mean_squared_error`. Internally, a `squared_error` operation computes the

element-wise square of the difference between `predictions` and `labels`. Then