1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 package de.fu_berlin.ties.eval;
23
24 import de.fu_berlin.ties.io.BaseStorable;
25 import de.fu_berlin.ties.io.FieldMap;
26 import de.fu_berlin.ties.util.Util;
27
28 /***
29 * This class manages and updates evaluation results, calculating
30 * <em>precision (P)</em>, <em>recall (R)</em> and <em>F-measure (F)</em>.
31 * It counts <em>true positives (tp)</em>, <em>false negatives (tn)</em> and
32 * <em>false positives (fp)</em>.
33 *
34 * <p>The metrics are defines as follows (cf. Manning and Schütze (1999):
35 * <em>Foundations of Statistical Natural Language Processing</em>,
36 * p. 268f):
37 *
38 * <dl>
39 * <dt>Precision <em>P</em> =</dt>
40 * <dd><em>tp</em> / (<em>tp</em> + <em>fp</em>)</dd>
41 * <dt>Recall <em>R</em> =</dt>
42 * <dd><em>tp</em> / (<em>tp</em> + <em>fn</em>)</dd>
43 * <dt>F-measure F =</dt>
44 * <dd>1 / (<em>alpha</em> * (1/<em>P</em>) +
45 * (1-<em>alpha</em>) * (1/<em>R</em>))<br>
46 * We define <em>F</em> to be 0 if <em>P</em> = 0 or <em>R</em> = 0 (not
47 * discussed by Manning and Schütze).<br>
48 * The factor <em>alpha</em> defines the weighting of precision and recall;
49 * <em>P</em> and <em>R</em> are weighted equal if <em>alpha</em> = 0.5
50 * (the default value).</dd>
51 * </dl>
52 *
53 * <p>Instances of this class are not thread-safe and must be synchronized
54 * externally, if required.
55 *
56 * @author Christian Siefkes
57 * @version $Revision: 1.8 $, $Date: 2006/10/21 16:04:11 $, $Author: siefkes $
58 */
59 public class FMetrics extends BaseStorable implements FMetricsView {
60
61 /***
62 * Serialization key for true positives.
63 */
64 public static final String KEY_TRUE_POS = "True Positives";
65
66 /***
67 * Serialization key for false negatives.
68 */
69 public static final String KEY_FALSE_NEG = "False Negatives";
70
71 /***
72 * Serialization key for false positives.
73 */
74 public static final String KEY_FALSE_POS = "False Positives";
75
76 /***
77 * Serialization key for the precision.
78 */
79 public static final String KEY_PRECISION = "Precision";
80
81 /***
82 * Serialization key for the recall.
83 */
84 public static final String KEY_RECALL = "Recall";
85
86 /***
87 * Serialization key for the F1 measure.
88 */
89 public static final String KEY_F1_MEASURE = "F1-Measure";
90
91 /***
92 * The number of true positives (correct recognitions).
93 */
94 private long truePos = 0;
95
96 /***
97 * The number of false negatives (false rejections).
98 */
99 private long falseNeg = 0;
100
101 /***
102 * The number of false positives (false acceptances).
103 */
104 private long falsePos = 0;
105
106 /***
107 * Cached precision value, stored for efficiency; <code>-1</code> if not
108 * known or out-of-date.
109 */
110 private double cachedPrecision = -1.0;
111
112 /***
113 * Cached recall value, stored for efficiency; <code>-1</code> if not
114 * known or out-of-date.
115 */
116 private double cachedRecall = -1.0;
117
118 /***
119 * Creates a new empty instance.
120 */
121 public FMetrics() {
122 super();
123 }
124
125 /***
126 * Creates a new instance.
127 *
128 * @param input contains the initial number of true and false positives and
129 * false negatives
130 * @throws IllegalArgumentException if at least one of the input value is
131 * negative
132 */
133 public FMetrics(final EvalInput input) throws IllegalArgumentException {
134 this(input.getTruePos(), input.getFalseNeg(), input.getFalsePos());
135 }
136
137 /***
138 * Creates a new instance from a field map, fulfilling the
139 * {@link de.fu_berlin.ties.io.Storable} contract.
140 *
141 * @param fieldMap map containing the serialized fields
142 * @throws IllegalArgumentException if at least one of the parameters is
143 * negative or missing
144 */
145 public FMetrics(final FieldMap fieldMap) throws IllegalArgumentException {
146 this(Util.asLong(fieldMap.get(KEY_TRUE_POS)),
147 Util.asLong(fieldMap.get(KEY_FALSE_NEG)),
148 Util.asLong(fieldMap.get(KEY_FALSE_POS)));
149 }
150
151 /***
152 * Creates a new instance.
153 *
154 * @param initTruePos the initial number of true positives
155 * @param initFalseNeg the initial number of false negatives
156 * @param initFalsePos the initial number of false positives
157 * @throws IllegalArgumentException if at least one of the parameters is
158 * negative
159 */
160 public FMetrics(final long initTruePos, final long initFalseNeg,
161 final long initFalsePos) throws IllegalArgumentException {
162 this();
163 checkValues(initTruePos, initFalseNeg, initFalsePos);
164 truePos = initTruePos;
165 falseNeg = initFalseNeg;
166 falsePos = initFalsePos;
167 }
168
169 /***
170 * Checks the provided parameters, throwing an exception if at least one
171 * of them is negative.
172 *
173 * @param theTruePos the number of true positives to check
174 * @param theFalseNeg the number of false negatives to check
175 * @param theFalsePos the number of false positives to check
176 * @throws IllegalArgumentException if at least one of the parameters is
177 * negative
178 */
179 private void checkValues(final long theTruePos,
180 final long theFalseNeg, final long theFalsePos)
181 throws IllegalArgumentException {
182 if (theTruePos < 0) {
183 throw new IllegalArgumentException(
184 "Negative number of true positives: " + theTruePos);
185 }
186 if (theFalseNeg < 0) {
187 throw new IllegalArgumentException(
188 "Negative number of false negatives: " + theFalseNeg);
189 }
190 if (theFalsePos < 0) {
191 throw new IllegalArgumentException(
192 "Negative number of false positives: " + theFalsePos);
193 }
194 }
195
196 /***
197 * Helper method that resets the cached precision + recall values. Must be
198 * called whenever input values change.
199 */
200 private void clearCache() {
201 cachedPrecision = -1.0;
202 cachedRecall = -1.0;
203 }
204
205 /***
206 * Returns the number of false negatives (false rejections).
207 * @return the number of false negatives
208 */
209 public final long getFalseNeg() {
210 return falseNeg;
211 }
212
213 /***
214 * Returns the number of false positives (false acceptances).
215 * @return the number of false positives
216 */
217 public final long getFalsePos() {
218 return falsePos;
219 }
220
221 /***
222 * Returns the F-measure, setting <em>alpha</em> = 0.5 so <em>P</em> and
223 * <em>R</em> are weighted equal ("F1 measure").
224 * F1 = (2 * <em>P</em> * <em>R</em>) / (<em>P</em> + <em>R</em>).
225 *
226 * @return the F1 measure; a value in the range from 0.0 to 1.0
227 */
228 public final double getF1Measure() {
229 return getFMeasure(0.5);
230 }
231
232 /***
233 * Returns the F-measure: F = 1 / (<em>alpha</em> * (1/<em>P</em>)
234 * + (1-<em>alpha</em>) * (1/<em>R</em>)). <br>
235 * <em>F</em> is defined to be 0 if <em>P</em> = 0 or <em>R</em> = 0.
236 *
237 * @param alpha a factor in the range from 0.0 to 1.0 defining the weighting
238 * of precision and recall
239 * @return the F-measure; a value in the range from 0.0 to 1.0
240 * @throws IllegalArgumentException if <code>alpha</code> is smaller than
241 * 0.0 or larger than 1.0
242 */
243 public final double getFMeasure(final double alpha)
244 throws IllegalArgumentException {
245 if ((alpha < 0.0) || (alpha > 1.0)) {
246 throw new IllegalArgumentException(
247 "alpha factor outside of 0.0 to 1.0 range: " + alpha);
248 }
249 final double inverted = alpha * (1.0 / getPrecision())
250 + (1.0 - alpha) * (1.0 / getRecall());
251
252 if (Double.isNaN(inverted)) {
253
254 return 0.0;
255 } else {
256
257 return 1.0 / inverted;
258 }
259 }
260
261 /***
262 * Returns the precision: <em>P</em> =
263 * <em>tp</em> / (<em>tp</em> + <em>fp</em>).
264 * @return the precision; a value in the range from 0.0 to 1.0
265 */
266 public final double getPrecision() {
267 if (cachedPrecision < 0.0) {
268
269 cachedPrecision = ((double) truePos) / (truePos + falsePos);
270 }
271
272 return cachedPrecision;
273 }
274
275 /***
276 * Returns the recall: <em>R</em> =
277 * <em>tp</em> / (<em>tp</em> + <em>fn</em>).
278 * @return the precision; a value in the range from 0.0 to 1.0
279 */
280 public final double getRecall() {
281 if (cachedRecall < 0.0) {
282
283 cachedRecall = ((double) truePos) / (truePos + falseNeg);
284 }
285
286 return cachedRecall;
287 }
288
289 /***
290 * Returns the number of true positives (correct recognitions).
291 * @return the number of true positives
292 */
293 public final long getTruePos() {
294 return truePos;
295 }
296
297 /***
298 * Increases the number of false negatives by 1.
299 */
300 public final void incFalseNeg() {
301 clearCache();
302 falseNeg++;
303 }
304
305 /***
306 * Increases the number of false positives by 1.
307 */
308 public final void incFalsePos() {
309 clearCache();
310 falsePos++;
311 }
312
313 /***
314 * Increases the number of true positives by 1.
315 */
316 public final void incTruePos() {
317 clearCache();
318 truePos++;
319 }
320
321 /***
322 * Stores all relevant fields of this object in a field map for
323 * serialization. An equivalent object can be created by calling
324 * {@link de.fu_berlin.ties.io.FieldMap#createObject(Class)} on the created
325 * field map. The calculated values precision, recall, and F-measure are
326 * also stored (they are ignored when
327 * {@linkplain #FMetrics(FieldMap) deserializing} a stored instance).
328 *
329 * @return the created field map
330 */
331 public FieldMap storeFields() {
332 final FieldMap result = new FieldMap();
333 result.put(KEY_PRECISION, new Double(getPrecision()));
334 result.put(KEY_RECALL, new Double(getRecall()));
335 result.put(KEY_F1_MEASURE, new Double(getF1Measure()));
336 result.put(KEY_TRUE_POS, new Long(truePos));
337 result.put(KEY_FALSE_NEG, new Long(falseNeg));
338 result.put(KEY_FALSE_POS, new Long(falsePos));
339 return result;
340 }
341
342 /***
343 * Updates the statistics, increasing the stored values as specified.
344 *
345 * @param input contains the number of true and false positives and
346 * false negatives to add
347 * @throws IllegalArgumentException if at least one of the input values is
348 * negative
349 */
350 public final void update(final EvalInput input) throws
351 IllegalArgumentException {
352 update(input.getTruePos(), input.getFalseNeg(), input.getFalsePos());
353 }
354
355 /***
356 * Updates the statistics, increasing the stored values as specified.
357 *
358 * @param addTruePos the number of new true positives to add
359 * @param addFalseNeg the number of new false negatives to add
360 * @param addFalsePos the number of new false positives to add
361 * @throws IllegalArgumentException if at least one of the parameters is
362 * negative
363 */
364 public void update(final long addTruePos, final long addFalseNeg,
365 final long addFalsePos) throws IllegalArgumentException {
366 checkValues(addTruePos, addFalseNeg, addFalsePos);
367 clearCache();
368 truePos += addTruePos;
369 falseNeg += addFalseNeg;
370 falsePos += addFalsePos;
371 }
372
373 }