看到一個使用 tf 實現的 precision、recall 和 f1，仔細看發現這個實現里 micro-precision、micro-recall、micro-f1 相等，以前從沒認真想過這個問題，但是仔細一想還真是這樣，於是趕緊用 google 搜了一下，發現還有篇博客介紹了並舉例子驗證。
三指標的源碼如下：
"""Multiclass"""
__author__ = "Guillaume Genthial"
import numpy as np
import tensorflow as tf
from tensorflow.python.ops.metrics_impl import _streaming_confusion_matrix
def precision(labels, predictions, num_classes, pos_indices=None,
weights=None, average='micro'):
"""Multi-class precision metric for Tensorflow
Parameters
----------
labels : Tensor of tf.int32 or tf.int64
The true labels
predictions : Tensor of tf.int32 or tf.int64
The predictions, same shape as labels
num_classes : int
The number of classes
pos_indices : list of int, optional
The indices of the positive classes, default is all
weights : Tensor of tf.int32, optional
Mask, must be of compatible shape with labels
average : str, optional
'micro': counts the total number of true positives, false
positives, and false negatives for the classes in
`pos_indices` and infer the metric from it.
'macro': will compute the metric separately for each class in
`pos_indices` and average. Will not account for class
imbalance.
'weighted': will compute the metric separately for each class in
`pos_indices` and perform a weighted average by the total
number of true labels for each class.
Returns
-------
tuple of (scalar float Tensor, update_op)
"""
cm, op = _streaming_confusion_matrix(
labels, predictions, num_classes, weights)
pr, _, _ = metrics_from_confusion_matrix(
cm, pos_indices, average=average)
op, _, _ = metrics_from_confusion_matrix(
op, pos_indices, average=average)
return (pr, op)
def recall(labels, predictions, num_classes, pos_indices=None, weights=None,
average='micro'):
"""Multi-class recall metric for Tensorflow
Parameters
----------
labels : Tensor of tf.int32 or tf.int64
The true labels
predictions : Tensor of tf.int32 or tf.int64
The predictions, same shape as labels
num_classes : int
The number of classes
pos_indices : list of int, optional
The indices of the positive classes, default is all
weights : Tensor of tf.int32, optional
Mask, must be of compatible shape with labels
average : str, optional
'micro': counts the total number of true positives, false
positives, and false negatives for the classes in
`pos_indices` and infer the metric from it.
'macro': will compute the metric separately for each class in
`pos_indices` and average. Will not account for class
imbalance.
'weighted': will compute the metric separately for each class in
`pos_indices` and perform a weighted average by the total
number of true labels for each class.
Returns
-------
tuple of (scalar float Tensor, update_op)
"""
cm, op = _streaming_confusion_matrix(
labels, predictions, num_classes, weights)
_, re, _ = metrics_from_confusion_matrix(
cm, pos_indices, average=average)
_, op, _ = metrics_from_confusion_matrix(
op, pos_indices, average=average)
return (re, op)
def f1(labels, predictions, num_classes, pos_indices=None, weights=None,
average='micro'):
return fbeta(labels, predictions, num_classes, pos_indices, weights,
average)
def fbeta(labels, predictions, num_classes, pos_indices=None, weights=None,
average='micro', beta=1):
"""Multi-class fbeta metric for Tensorflow
Parameters
----------
labels : Tensor of tf.int32 or tf.int64
The true labels
predictions : Tensor of tf.int32 or tf.int64
The predictions, same shape as labels
num_classes : int
The number of classes
pos_indices : list of int, optional
The indices of the positive classes, default is all
weights : Tensor of tf.int32, optional
Mask, must be of compatible shape with labels
average : str, optional
'micro': counts the total number of true positives, false
positives, and false negatives for the classes in
`pos_indices` and infer the metric from it.
'macro': will compute the metric separately for each class in
`pos_indices` and average. Will not account for class
imbalance.
'weighted': will compute the metric separately for each class in
`pos_indices` and perform a weighted average by the total
number of true labels for each class.
beta : int, optional
Weight of precision in harmonic mean
Returns
-------
tuple of (scalar float Tensor, update_op)
"""
cm, op = _streaming_confusion_matrix(
labels, predictions, num_classes, weights)
_, _, fbeta = metrics_from_confusion_matrix(
cm, pos_indices, average=average, beta=beta)
_, _, op = metrics_from_confusion_matrix(
op, pos_indices, average=average, beta=beta)
return (fbeta, op)
def safe_div(numerator, denominator):
"""Safe division, return 0 if denominator is 0"""
numerator, denominator = tf.cast(numerator, tf.float32), tf.cast(denominator, tf.float32)
zeros = tf.zeros_like(numerator, dtype=numerator.dtype)
denominator_is_zero = tf.equal(denominator, zeros)
return tf.where(denominator_is_zero, zeros, numerator / denominator)
def pr_re_fbeta(cm, pos_indices, beta=1):
"""Uses a confusion matrix to compute precision, recall and fbeta"""
num_classes = cm.shape[0]
neg_indices = [i for i in range(num_classes) if i not in pos_indices]
cm_mask = np.ones([num_classes, num_classes])
cm_mask[neg_indices, neg_indices] = 0
diag_sum = tf.reduce_sum(tf.linalg.diag_part(cm * cm_mask))
cm_mask = np.ones([num_classes, num_classes])
cm_mask[:, neg_indices] = 0
tot_pred = tf.reduce_sum(cm * cm_mask)
cm_mask = np.ones([num_classes, num_classes])
cm_mask[neg_indices, :] = 0
tot_gold = tf.reduce_sum(cm * cm_mask)
pr = safe_div(diag_sum, tot_pred)
re = safe_div(diag_sum, tot_gold)
fbeta = safe_div((1. + beta**2) * pr * re, beta**2 * pr + re)
return pr, re, fbeta
def metrics_from_confusion_matrix(cm, pos_indices=None, average='micro',
beta=1):
"""Precision, Recall and F1 from the confusion matrix
Parameters
----------
cm : tf.Tensor of type tf.int32, of shape (num_classes, num_classes)
The streaming confusion matrix.
pos_indices : list of int, optional
The indices of the positive classes
beta : int, optional
Weight of precision in harmonic mean
average : str, optional
'micro', 'macro' or 'weighted'
"""
num_classes = cm.shape[0]
if pos_indices is None:
pos_indices = [i for i in range(num_classes)]
if average == 'micro':
return pr_re_fbeta(cm, pos_indices, beta)
elif average in {'macro', 'weighted'}:
precisions, recalls, fbetas, n_golds = [], [], [], []
for idx in pos_indices:
pr, re, fbeta = pr_re_fbeta(cm, [idx], beta)
precisions.append(pr)
recalls.append(re)
fbetas.append(fbeta)
cm_mask = np.zeros([num_classes, num_classes])
cm_mask[idx, :] = 1
n_golds.append(tf.cast(tf.reduce_sum(cm * cm_mask), tf.float32))
if average == 'macro':
pr = tf.reduce_mean(precisions)
re = tf.reduce_mean(recalls)
fbeta = tf.reduce_mean(fbetas)
return pr, re, fbeta
if average == 'weighted':
n_gold = tf.reduce_sum(n_golds)
pr_sum = sum(p * n for p, n in zip(precisions, n_golds))
pr = safe_div(pr_sum, n_gold)
re_sum = sum(r * n for r, n in zip(recalls, n_golds))
re = safe_div(re_sum, n_gold)
fbeta_sum = sum(f * n for f, n in zip(fbetas, n_golds))
fbeta = safe_div(fbeta_sum, n_gold)
return pr, re, fbeta
else:
raise NotImplementedError()
以上是三個指標函數的實現，涉及到一個不在此文件中的輔助函數：_streaming_confusion_matrix，這個函數的源碼如下：
def _streaming_confusion_matrix(labels, predictions, num_classes, weights=None):
"""Calculate a streaming confusion matrix.
Calculates a confusion matrix. For estimation over a stream of data,
the function creates an `update_op` operation.
Args:
labels: A `Tensor` of ground truth labels with shape [batch size] and of
type `int32` or `int64`. The tensor will be flattened if its rank > 1.
predictions: A `Tensor` of prediction results for semantic labels, whose
shape is [batch size] and type `int32` or `int64`. The tensor will be
flattened if its rank > 1.
num_classes: The possible number of labels the prediction task can
have. This value must be provided, since a confusion matrix of
dimension = [num_classes, num_classes] will be allocated.
weights: Optional `Tensor` whose rank is either 0, or the same rank as
`labels`, and must be broadcastable to `labels` (i.e., all dimensions must
be either `1`, or the same as the corresponding `labels` dimension).
Returns:
total_cm: A `Tensor` representing the confusion matrix.
update_op: An operation that increments the confusion matrix.
"""
# Local variable to accumulate the predictions in the confusion matrix.
total_cm = metric_variable(
[num_classes, num_classes], dtypes.float64, name='total_confusion_matrix')
# Cast the type to int64 required by confusion_matrix_ops.
predictions = math_ops.cast(predictions, dtypes.int64)
labels = math_ops.cast(labels, dtypes.int64)
num_classes = math_ops.cast(num_classes, dtypes.int64)
# Flatten the input if its rank > 1.
if predictions.get_shape().ndims > 1:
predictions = array_ops.reshape(predictions, [-1])
if labels.get_shape().ndims > 1:
labels = array_ops.reshape(labels, [-1])
if (weights is not None) and (weights.get_shape().ndims > 1):
weights = array_ops.reshape(weights, [-1])
# Accumulate the prediction to current confusion matrix.
current_cm = confusion_matrix.confusion_matrix(
labels, predictions, num_classes, weights=weights, dtype=dtypes.float64)
update_op = state_ops.assign_add(total_cm, current_cm)
return total_cm, update_op
主要看它的兩個輸出有什么區別，這兩個輸出分別傳入到 metrics_from_confusion_matrix 導致最后三個指標最后輸出的都是一個二元組。查看 state_ops.assign_add 的源碼：
@tf_export(v1=["assign_add"])
def assign_add(ref, value, use_locking=None, name=None):
"""Update `ref` by adding `value` to it.
This operation outputs "ref" after the update is done.
This makes it easier to chain operations that need to use the reset value.
Unlike `tf.math.add`, this op does not broadcast. `ref` and `value` must have
the same shape.
Args:
ref: A mutable `Tensor`. Must be one of the following types: `float32`,
`float64`, `int64`, `int32`, `uint8`, `uint16`, `int16`, `int8`,
`complex64`, `complex128`, `qint8`, `quint8`, `qint32`, `half`. Should be
from a `Variable` node.
value: A `Tensor`. Must have the same shape and dtype as `ref`. The value to
be added to the variable.
use_locking: An optional `bool`. Defaults to `False`. If True, the addition
will be protected by a lock; otherwise the behavior is undefined, but may
exhibit less contention.
name: A name for the operation (optional).
Returns:
Same as "ref". Returned as a convenience for operations that want
to use the new value after the variable has been updated.
"""
if ref.dtype._is_ref_dtype:
return gen_state_ops.assign_add(
ref, value, use_locking=use_locking, name=name)
return ref.assign_add(value)
這倆貌似是一個東西，不太清楚返回的元組中兩個數字相等有什么作用。先放下這個問題，總之我們得到了一個混淆矩陣，維度為 num_classed * num_classes。接下來主要分析函數metrics_from_confusion_matrix這個函數主要根據不同的 average 模式計算三指標，主要是其中的函數pr_re_fbeta。先假設我們通過函數pr_re_fbeta得到了三個指標的結果，如果是 micro 模式，結果直接就是函數pr_re_fbeta的結果，傳入的參數pos_indices和其他兩種模式不同，待會看具體實現。如果是 macro 和 weighted 模式，通過循環將每一個類分別作為正類計算得到三個指標，其中 macro 直接對 num_classes 平均得到最后的結果，這個計算方式其實和周志華老師的《機器學習》一書中的計算方式不太一致，此書中是先計算出 num_classes 個 precision 和 recall，然后將其做平均，然后使用平均 precision 和平均 recall 計算最終的 f1；weighted 模式還需要使用每種類別的個數作為權重，也就是使用加權平均。
然后看函數pr_re_fbeta，先看 macro 模式和 weighted 模式調用的過程，cm 是前面的得到的混淆矩陣，beta 的含義不變，pos_indices 這里是一個只包含一個元素的列表，即當前循環的正類，假設 cm 如下：
\[ \begin{matrix} 4 & 0 & 2 \\ 1 & 5 & 0 \\ 3 & 2 & 9 \end{matrix} \]
\(cm_{ij}\) 表示實際為 \(i\) 類，預測為 \(j\) 類的樣本數。neg_indices 表示當前負類的所有序號，假設當前正類為 1，那么 0、2 為負類，將得到下面的 cm_mask:
\[ \begin{matrix} 0 & 1 & 1 \\ 1 & 1 & 1 \\ 1 & 1 & 0 \end{matrix} \]
將 cm 和 cm_mask 按位相乘后取出對角線元素並求和，得到的剛好就是 true positive；下面繼續生成 cm_mask：
\[ \begin{matrix} 0 & 1 & 0 \\ 0 & 1 & 0 \\ 0 & 1 & 0 \end{matrix} \]
將 cm 和 cm_mask 按位相乘后相加得到的就是所有預測為 1 類別的樣本數，即 true positive + false positive；繼續生成 cm_mask：
\[ \begin{matrix} 0 & 0 & 0 \\ 1 & 1 & 1 \\ 0 & 0 & 0 \end{matrix} \]
將 cm 和 cm_mask 按位相乘后相加得到的就是所有真實為 1 類別的樣本數，即 true positive + false negative。
接下來就可以計算出 \(i\) 為正類的時候的三指標值。
當為 micro 模式時，傳入的 pos_indices 是所有的類別編號，那么 neg_indices 直接為空，所有的 mask 都失效了，首先是 true positive 的計算，這還好理解，就是對角線上所有元素求和，由於 mask 失效，就是所有類別被正確預測的樣本數之和；乍一看上去讓人有些不好理解的是 true positive + false positive 和 true positive + false negative 的計算，它們都等於 cm 中所有元素的和，這里提供一個定性的理解：假設元素 \(cm_{21}\) 等於 3，首先明確它的真實類別是 2 ，預測類別是 1，對於類別 2 來說，它是一個 false negative，而對於類別 1 來說，它是一個 false positive，也就是說一個預測錯誤的樣本向 false positive 和 false negative 的貢獻一致，那么最終這二者必定相等，也就導致 precision 和 recall 相等，由 f1 的計算式可知也會相等。
在面對不平衡數據時，這三種模式哪種會更好呢，這應該視問題而定，如果我們更關注少數類，那么應該使用 macro 模式。因為在 micro 模式中，即便少數類的 true positive 為 0，在進行 每種類的 true positive 加和之后，仍然可以得到一個較大的數值，得到一個還不錯的分數；而 weighted 中，會由於樣本少權重小，即便錯了也不會對指標有大的影響。在 macro 中，平均操作在每種類別的 precision、recall 和 f1 計算完之后，所以少數類的分類效果不好的話會影響最終的指標。
這篇博客進行了定量的驗證。
參考：