View Javadoc

1   /*
2    * Copyright (C) 2004 Christian Siefkes <christian@siefkes.net>.
3    * Development of this software is supported by the German Research Society,
4    * Berlin-Brandenburg Graduate School in Distributed Information Systems
5    * (DFG grant no. GRK 316).
6    *
7    * This library is free software; you can redistribute it and/or
8    * modify it under the terms of the GNU Lesser General Public
9    * License as published by the Free Software Foundation; either
10   * version 2.1 of the License, or (at your option) any later version.
11   *
12   * This library is distributed in the hope that it will be useful,
13   * but WITHOUT ANY WARRANTY; without even the implied warranty of
14   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15   * Lesser General Public License for more details.
16   *
17   * You should have received a copy of the GNU Lesser General Public
18   * License along with this library; if not, visit
19   * http://www.gnu.org/licenses/lgpl.html or write to the Free Software
20   * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
21   */
22  package de.fu_berlin.ties.filter;
23  
24  import java.util.Iterator;
25  import java.util.Set;
26  
27  import org.dom4j.Element;
28  
29  import de.fu_berlin.ties.eval.FMetrics;
30  import de.fu_berlin.ties.eval.FMetricsView;
31  
32  /***
33   * A static class that allows to evaluate the precision, recall, and F1 measure
34   * of an {@linkplain de.fu_berlin.ties.filter.ElementFilter element filter}.
35   * No instances of this class can be created, only the static members
36   * should be used.
37   *
38   * @author Christian Siefkes
39   * @version $Revision: 1.2 $, $Date: 2004/09/01 16:35:45 $, $Author: siefkes $
40   */
41  public final class FilterEvaluator {
42  
43      /***
44       * Private constructor prevents creation of instances.
45       */
46      private FilterEvaluator() {
47          super();
48      }
49  
50      /***
51       * Evaluates precision, recall, and F1 measure of the last walk performed
52       * by a filtering token woken. Delegates to
53       * {@link #evaluate(EmbeddingElements, Set, Set)}.
54       *
55       * @param embeddingElements the correct set of embedding elements
56       * @param filteringWalker the filtering token walker to evaluate
57       * @return the calculated statistics
58       */
59      public static FMetricsView evaluate(
60              final EmbeddingElements embeddingElements,
61              final FilteringTokenWalker filteringWalker) {
62          return evaluate(embeddingElements,
63                  filteringWalker.getAcceptedElements(),
64                  filteringWalker.getRejectedElements());
65      }
66  
67      /***
68       * Evaluates precision, recall, and F1 measure of an element filter by
69       * comparing the accepted and rejected elements of the filter with the
70       * correct set of <code>embeddingElements</code>.
71       *
72       * @param embeddingElements the correct set of embedding elements
73       * @param acceptedElements the set of elements accepted by the filter
74       * @param rejectedElements the set of elements rejected by the filter
75       * @return the calculated statistics
76       */
77      public static FMetricsView evaluate(
78              final EmbeddingElements embeddingElements,
79              final Set acceptedElements, final Set rejectedElements) {
80          final FMetrics result = new FMetrics();
81          Element currentElement;
82  
83          /* final Set falsePos = new HashSet(); // only for testing
84          final Set falseNeg = new HashSet(); */
85  
86          // iterate accepted elements
87          final Iterator acceptedIter = acceptedElements.iterator();
88          while (acceptedIter.hasNext()) {
89              currentElement = (Element) acceptedIter.next();
90  
91              if (embeddingElements.containsExtraction(currentElement)) {
92                  // acceptance was correct: true positive
93                  result.incTruePos();
94              } else {
95                  // acceptance was erroneous: false positive
96                  result.incFalsePos();
97                  //falsePos.add(DOMUtils.showElement(currentElement));
98              }
99          }
100 
101         // iterate rejected elements
102         final Iterator rejectedIter = rejectedElements.iterator();
103         while (rejectedIter.hasNext()) {
104             currentElement = (Element) rejectedIter.next();
105 
106             if (embeddingElements.containsExtraction(currentElement)) {
107                 // reject was erroneous: false negative
108                 result.incFalseNeg();
109                 //falseNeg.add(DOMUtils.showElement(currentElement));
110             }
111             // otherwise reject was correct: true negative (not counted)
112         }
113 
114         /* if (!falsePos.isEmpty()) {
115             Util.LOG.debug("False positives: " + falsePos);
116         }
117         if (!falseNeg.isEmpty()) {
118             Util.LOG.debug("False negatives: " + falseNeg);
119         } */
120 
121         return result;
122     }
123 
124 }