View Javadoc

1   /*
2    * Copyright (C) 2004 Christian Siefkes <christian@siefkes.net>.
3    * Development of this software is supported by the German Research Society,
4    * Berlin-Brandenburg Graduate School in Distributed Information Systems
5    * (DFG grant no. GRK 316).
6    *
7    * This library is free software; you can redistribute it and/or
8    * modify it under the terms of the GNU Lesser General Public
9    * License as published by the Free Software Foundation; either
10   * version 2.1 of the License, or (at your option) any later version.
11   *
12   * This library is distributed in the hope that it will be useful,
13   * but WITHOUT ANY WARRANTY; without even the implied warranty of
14   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15   * Lesser General Public License for more details.
16   *
17   * You should have received a copy of the GNU Lesser General Public
18   * License along with this library; if not, visit
19   * http://www.gnu.org/licenses/lgpl.html or write to the Free Software
20   * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
21   */
22  package de.fu_berlin.ties.eval;
23  
24  import de.fu_berlin.ties.io.BaseStorable;
25  import de.fu_berlin.ties.io.FieldMap;
26  import de.fu_berlin.ties.util.Util;
27  
28  /***
29   * This class manages and updates evaluation results, calculating
30   * <em>precision (P)</em>, <em>recall (R)</em> and <em>F-measure (F)</em>.
31   * It counts <em>true positives (tp)</em>, <em>false negatives (tn)</em> and
32   * <em>false positives (fp)</em>.
33   *
34   * <p>The metrics are defines as follows (cf. Manning and Schütze (1999):
35   * <em>Foundations of Statistical Natural Language Processing</em>,
36   * p.&nbsp;268f):
37   *
38   * <dl>
39   * <dt>Precision <em>P</em> =</dt>
40   * <dd><em>tp</em> / (<em>tp</em> + <em>fp</em>)</dd>
41   * <dt>Recall <em>R</em> =</dt>
42   * <dd><em>tp</em> / (<em>tp</em> + <em>fn</em>)</dd>
43   * <dt>F-measure F =</dt>
44   * <dd>1 / (<em>alpha</em> * (1/<em>P</em>) +
45   *         (1-<em>alpha</em>) * (1/<em>R</em>))<br>
46   * We define <em>F</em> to be 0 if <em>P</em> = 0 or <em>R</em> = 0 (not
47   * discussed by Manning and Schütze).<br>
48   * The factor <em>alpha</em> defines the weighting of precision and recall;
49   * <em>P</em> and <em>R</em> are weighted equal if <em>alpha</em> = 0.5
50   * (the default value).</dd>
51   * </dl>
52   *
53   * <p>Instances of this class are not thread-safe and must be synchronized
54   * externally, if required.
55   *
56   * @author Christian Siefkes
57   * @version $Revision: 1.5 $, $Date: 2004/10/11 15:30:41 $, $Author: siefkes $
58   */
59  public class FMetrics extends BaseStorable implements FMetricsView {
60  
61      /***
62       * Serialization key for true positives.
63       */
64      public static final String KEY_TRUE_POS = "True Positives";
65  
66      /***
67       * Serialization key for false negatives.
68       */
69      public static final String KEY_FALSE_NEG = "False Negatives";
70  
71      /***
72       * Serialization key for false positives.
73       */
74      public static final String KEY_FALSE_POS = "False Positives";
75  
76      /***
77       * Serialization key for the precision.
78       */
79      public static final String KEY_PRECISION = "Precision";
80  
81      /***
82       * Serialization key for the recall.
83       */
84      public static final String KEY_RECALL = "Recall";
85  
86      /***
87       * Serialization key for the F1 measure.
88       */
89      public static final String KEY_F1_MEASURE = "F1-Measure";
90  
91      /***
92       * The number of true positives (correct recognitions).
93       */
94      private long truePos = 0;
95  
96      /***
97       * The number of false negatives (false rejections).
98       */
99      private long falseNeg = 0;
100 
101     /***
102      * The number of false positives (false acceptances).
103      */
104     private long falsePos = 0;
105 
106     /***
107      * Cached precision value, stored for efficiency; <code>-1</code> if not
108      * known or out-of-date.
109      */
110     private double cachedPrecision = -1.0;
111 
112     /***
113      * Cached recall value, stored for efficiency; <code>-1</code> if not
114      * known or out-of-date.
115      */
116     private double cachedRecall = -1.0;
117 
118     /***
119      * Creates a new empty instance.
120      */
121     public FMetrics() {
122         super();
123     }
124 
125     /***
126      * Creates a new instance.
127      *
128      * @param input contains the initial number of true and false positives and
129      * false negatives
130      * @throws IllegalArgumentException if at least one of the input value is
131      * negative
132      */
133     public FMetrics(final EvalInput input) throws IllegalArgumentException {
134         this(input.getTruePos(), input.getFalseNeg(), input.getFalsePos());
135     }
136 
137     /***
138      * Creates a new instance from a field map, fulfilling the
139      * {@link de.fu_berlin.ties.io.Storable} contract.
140      *
141      * @param fieldMap map containing the serialized fields
142      * @throws IllegalArgumentException if at least one of the parameters is
143      * negative or missing
144      */
145     public FMetrics(final FieldMap fieldMap) throws IllegalArgumentException {
146         this(Util.asLong(fieldMap.get(KEY_TRUE_POS)),
147             Util.asLong(fieldMap.get(KEY_FALSE_NEG)),
148             Util.asLong(fieldMap.get(KEY_FALSE_POS)));
149     }
150 
151     /***
152      * Creates a new instance.
153      *
154      * @param initTruePos the initial number of true positives
155      * @param initFalseNeg the initial number of false negatives
156      * @param initFalsePos the initial number of false positives
157      * @throws IllegalArgumentException if at least one of the parameters is
158      * negative
159      */
160     public FMetrics(final long initTruePos, final long initFalseNeg,
161             final long initFalsePos) throws IllegalArgumentException {
162         this();
163         checkValues(initTruePos, initFalseNeg, initFalsePos);
164         truePos = initTruePos;
165         falseNeg = initFalseNeg;
166         falsePos = initFalsePos;
167     }
168 
169     /***
170      * Checks the provided parameters, throwing an exception if at least one
171      * of them is negative.
172      *
173      * @param theTruePos the number of true positives to check
174      * @param theFalseNeg the number of false negatives to check
175      * @param theFalsePos the number of false positives to check
176      * @throws IllegalArgumentException if at least one of the parameters is
177      * negative
178      */
179     private void checkValues(final long theTruePos,
180             final long theFalseNeg, final long theFalsePos)
181             throws IllegalArgumentException {
182         if (theTruePos < 0) {
183             throw new IllegalArgumentException(
184                 "Negative number of true positives: " + theTruePos);
185         }
186         if (theFalseNeg < 0) {
187             throw new IllegalArgumentException(
188                 "Negative number of false negatives: " + theFalseNeg);
189         }
190         if (theFalsePos < 0) {
191             throw new IllegalArgumentException(
192                 "Negative number of false positives: " + theFalsePos);
193         }
194     }
195 
196     /***
197      * Helper method that resets the cached precision + recall values. Must be
198      * called whenever input values change.
199      */
200     private void clearCache() {
201         cachedPrecision = -1.0;
202         cachedRecall = -1.0;
203     }
204 
205     /***
206      * Returns the number of false negatives (false rejections).
207      * @return the number of false negatives
208      */
209     public final long getFalseNeg() {
210         return falseNeg;
211     }
212 
213     /***
214      * Returns the number of false positives (false acceptances).
215      * @return the number of false positives
216      */
217     public final long getFalsePos() {
218         return falsePos;
219     }
220 
221     /***
222      * Returns the F-measure, setting <em>alpha</em> = 0.5 so <em>P</em> and
223      * <em>R</em> are weighted equal ("F1 measure").
224      * F1 = (2 * <em>P</em> * <em>R</em>) / (<em>P</em> + <em>R</em>).
225      *
226      * @return the F1 measure; a value in the range from 0.0 to 1.0
227      */
228     public final double getF1Measure() {
229         return getFMeasure(0.5);
230     }
231 
232     /***
233      * Returns the F-measure: F = 1 / (<em>alpha</em> * (1/<em>P</em>)
234      * + (1-<em>alpha</em>) * (1/<em>R</em>)). <br>
235      * <em>F</em> is defined to be 0 if <em>P</em> = 0 or <em>R</em> = 0.
236      *
237      * @param alpha a factor in the range from 0.0 to 1.0 defining the weighting
238      * of precision and recall
239      * @return the F-measure; a value in the range from 0.0 to 1.0
240      * @throws IllegalArgumentException if <code>alpha</code> is smaller than
241      * 0.0 or larger than 1.0
242      */
243     public final double getFMeasure(final double alpha)
244             throws IllegalArgumentException {
245         if ((alpha < 0.0) || (alpha > 1.0)) {
246             throw new IllegalArgumentException(
247                 "alpha factor outside of 0.0 to 1.0 range: " + alpha);
248         }
249         final double inverted = alpha * (1.0 / getPrecision())
250             + (1.0 - alpha) * (1.0 / getRecall());
251 
252         if (Double.isNaN(inverted)) {
253             // we define F-measure to be 0 if precision = 0 or recall = 0
254             return 0.0;
255         } else {
256             // inverted can never be 0
257             return 1.0 / inverted;
258         }
259     }
260 
261     /***
262      * Returns the precision: <em>P</em> =
263      * <em>tp</em> / (<em>tp</em> + <em>fp</em>).
264      * @return the precision; a value in the range from 0.0 to 1.0
265      */
266     public final double getPrecision() {
267         if (cachedPrecision < 0.0) {
268             // calculate and cache value
269             cachedPrecision = ((double) truePos) / (truePos + falsePos);
270         } // otherwise the value is already cached
271 
272         return cachedPrecision;
273     }
274 
275     /***
276      * Returns the recall: <em>R</em> =
277      * <em>tp</em> / (<em>tp</em> + <em>fn</em>).
278      * @return the precision; a value in the range from 0.0 to 1.0
279      */
280     public final double getRecall() {
281         if (cachedRecall < 0.0) {
282             // calculate and cache value
283             cachedRecall = ((double) truePos) / (truePos + falseNeg);
284         } // otherwise the value is already cached
285 
286         return cachedRecall;
287     }
288 
289     /***
290      * Returns the number of true positives (correct recognitions).
291      * @return the number of true positives
292      */
293     public final long getTruePos() {
294         return truePos;
295     }
296 
297     /***
298      * Increases the number of false negatives by 1.
299      */
300     public final void incFalseNeg() {
301         clearCache();
302         falseNeg++;
303     }
304 
305     /***
306      * Increases the number of false positives by 1.
307      */
308     public final void incFalsePos() {
309         clearCache();
310         falsePos++;
311     }
312 
313     /***
314      * Increases the number of true positives by 1.
315      */
316     public final void incTruePos() {
317         clearCache();
318         truePos++;
319     }
320 
321     /***
322      * Stores all relevant fields of this object in a field map for
323      * serialization. An equivalent object can be created by calling
324      * {@link de.fu_berlin.ties.io.FieldMap#createObject(Class)} on the created
325      * field map. The calculated values precision, recall, and F-measure are
326      * also stored (they are ignored when
327      * {@linkplain #FMetrics(FieldMap) deserializing} a stored instance).
328      *
329      * @return the created field map
330      */
331     public FieldMap storeFields() {
332         final FieldMap result = new FieldMap();
333         result.put(KEY_PRECISION, new Double(getPrecision()));
334         result.put(KEY_RECALL, new Double(getRecall()));
335         result.put(KEY_F1_MEASURE, new Double(getF1Measure()));
336         result.put(KEY_TRUE_POS, new Long(truePos));
337         result.put(KEY_FALSE_NEG, new Long(falseNeg));
338         result.put(KEY_FALSE_POS, new Long(falsePos));
339         return result;
340     }
341 
342     /***
343      * Updates the statistics, increasing the stored values as specified.
344      *
345      * @param input contains the number of true and false positives and
346      * false negatives to add
347      * @throws IllegalArgumentException if at least one of the input values is
348      * negative
349      */
350     public final void update(final EvalInput input) throws
351             IllegalArgumentException {
352         update(input.getTruePos(), input.getFalseNeg(), input.getFalsePos());
353     }
354 
355     /***
356      * Updates the statistics, increasing the stored values as specified.
357      *
358      * @param addTruePos the number of new true positives to add
359      * @param addFalseNeg the number of new false negatives to add
360      * @param addFalsePos the number of new false positives to add
361      * @throws IllegalArgumentException if at least one of the parameters is
362      * negative
363      */
364     public void update(final long addTruePos, final long addFalseNeg,
365             final long addFalsePos) throws IllegalArgumentException {
366         checkValues(addTruePos, addFalseNeg, addFalsePos);
367         clearCache();
368         truePos += addTruePos;
369         falseNeg += addFalseNeg;
370         falsePos += addFalsePos;
371     }
372 
373 }