View Javadoc

1   /*
2    * Copyright (C) 2005-2006 Christian Siefkes <christian@siefkes.net>.
3    * Development of this software is supported by the German Research Society,
4    * Berlin-Brandenburg Graduate School in Distributed Information Systems
5    * (DFG grant no. GRK 316).
6    *
7    * This program is free software; you can redistribute it and/or modify
8    * it under the terms of the GNU General Public License as published by
9    * the Free Software Foundation; either version 2 of the License, or
10   * (at your option) any later version.
11   *
12   * This program is distributed in the hope that it will be useful,
13   * but WITHOUT ANY WARRANTY; without even the implied warranty of
14   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15   * GNU General Public License for more details.
16   *
17   * You should have received a copy of the GNU General Public License
18   * along with this program; if not, visit
19   * http://www.gnu.org/licenses/gpl.html or write to the Free Software
20   * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
21   */
22  package de.fu_berlin.ties.eval;
23  
24  import java.io.File;
25  import java.io.IOException;
26  import java.io.Writer;
27  import java.util.Iterator;
28  import java.util.List;
29  
30  import de.fu_berlin.ties.CollectingProcessor;
31  import de.fu_berlin.ties.ContextMap;
32  import de.fu_berlin.ties.ProcessingException;
33  import de.fu_berlin.ties.TiesConfiguration;
34  import de.fu_berlin.ties.extract.AnswerBuilder;
35  import de.fu_berlin.ties.extract.EvaluatedExtractionContainer;
36  import de.fu_berlin.ties.extract.ExtractionContainer;
37  import de.fu_berlin.ties.extract.TargetStructure;
38  import de.fu_berlin.ties.io.FieldContainer;
39  import de.fu_berlin.ties.io.IOUtils;
40  import de.fu_berlin.ties.util.Util;
41  
42  /***
43   * Reads a set of files that must contain predictions and evaluates them
44   * against the corresponding answer keys (*.ans files).
45   * 
46   * @author Christian Siefkes
47   * @version $Revision: 1.4 $, $Date: 2006/10/21 16:04:11 $, $Author: siefkes $
48   */
49  public class PredictionEvaluator extends CollectingProcessor {
50  
51      /***
52       * Creates a new instance, using the standard configuration.
53       */
54      public PredictionEvaluator() {
55          this(TiesConfiguration.CONF);
56      }
57  
58      /***
59       * Creates a new instance.
60       *
61       * @param conf used to configure this instance
62       */
63      public PredictionEvaluator(final TiesConfiguration conf) {
64          super(conf);
65      }
66  
67  
68      /***
69       * {@inheritDoc}
70       */
71      public void process(final List<String> collected, final  ContextMap context)
72              throws IOException, ProcessingException {
73          final TargetStructure targetStruct = new TargetStructure(getConfig());
74          final EvaluatedExtractionContainer evalContainer =
75              new EvaluatedExtractionContainer(targetStruct, getConfig());
76  
77          String filename;
78          File file;
79          FieldContainer fContainer;
80          ExtractionContainer predictions, answerKeys;
81  
82          // process each prediction file
83          for (Iterator<String> iter = collected.iterator(); iter.hasNext();) {
84              filename = iter.next();
85              file = new File(filename);
86  
87              // read predictions
88              fContainer = FieldContainer.createFieldContainer(getConfig());
89              fContainer.read(IOUtils.openReader(file, getConfig()));
90              predictions = new ExtractionContainer(
91                      targetStruct, fContainer);
92  
93              // read corresponding answer keys and evaluate predictions
94              answerKeys = AnswerBuilder.readCorrespondingAnswerKeys(
95                      targetStruct, file, getConfig());
96              evalContainer.evaluateBatch(predictions, answerKeys,
97                      IOUtils.getBaseName(file));
98          }
99  
100         // serialize evaluated extractions
101         final File outDir = IOUtils.determineOutputDirectory(getConfig());
102         final String baseName = "All";
103         final FieldContainer storage =
104             FieldContainer.createFieldContainer(getConfig());
105         evalContainer.storeEntries(storage);
106         final File extFile = IOUtils.createOutFile(outDir, baseName, "ext");
107         final Writer extWriter = IOUtils.openWriter(extFile, getConfig());
108         try {
109             storage.store(extWriter);
110         } finally {
111             IOUtils.tryToClose(extWriter);
112         }
113 
114         // serialize metrics in same format
115         final File metricsFile = IOUtils.createOutFile(outDir, baseName,
116             MultiFMetrics.EXT_METRICS);
117         final Writer metricsWriter = IOUtils.openWriter(metricsFile,
118             getConfig());
119 
120         try {
121             final FieldContainer metricsStorage =
122                 FieldContainer.createFieldContainer(getConfig());
123             evalContainer.viewMetrics().storeEntries(metricsStorage);
124             metricsStorage.store(metricsWriter);
125             metricsWriter.flush();
126         } finally {
127             IOUtils.tryToClose(metricsWriter);
128         }
129 
130         Util.LOG.info("Stored evaluated extractions in " + extFile
131                 + " metrics in " + metricsFile);
132     }
133 
134 }