1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 package de.fu_berlin.ties.eval;
23
24 import java.io.File;
25 import java.io.IOException;
26 import java.io.Writer;
27 import java.util.Iterator;
28 import java.util.List;
29
30 import de.fu_berlin.ties.CollectingProcessor;
31 import de.fu_berlin.ties.ContextMap;
32 import de.fu_berlin.ties.ProcessingException;
33 import de.fu_berlin.ties.TiesConfiguration;
34 import de.fu_berlin.ties.extract.AnswerBuilder;
35 import de.fu_berlin.ties.extract.EvaluatedExtractionContainer;
36 import de.fu_berlin.ties.extract.ExtractionContainer;
37 import de.fu_berlin.ties.extract.TargetStructure;
38 import de.fu_berlin.ties.io.FieldContainer;
39 import de.fu_berlin.ties.io.IOUtils;
40 import de.fu_berlin.ties.util.Util;
41
42 /***
43 * Reads a set of files that must contain predictions and evaluates them
44 * against the corresponding answer keys (*.ans files).
45 *
46 * @author Christian Siefkes
47 * @version $Revision: 1.4 $, $Date: 2006/10/21 16:04:11 $, $Author: siefkes $
48 */
49 public class PredictionEvaluator extends CollectingProcessor {
50
51 /***
52 * Creates a new instance, using the standard configuration.
53 */
54 public PredictionEvaluator() {
55 this(TiesConfiguration.CONF);
56 }
57
58 /***
59 * Creates a new instance.
60 *
61 * @param conf used to configure this instance
62 */
63 public PredictionEvaluator(final TiesConfiguration conf) {
64 super(conf);
65 }
66
67
68 /***
69 * {@inheritDoc}
70 */
71 public void process(final List<String> collected, final ContextMap context)
72 throws IOException, ProcessingException {
73 final TargetStructure targetStruct = new TargetStructure(getConfig());
74 final EvaluatedExtractionContainer evalContainer =
75 new EvaluatedExtractionContainer(targetStruct, getConfig());
76
77 String filename;
78 File file;
79 FieldContainer fContainer;
80 ExtractionContainer predictions, answerKeys;
81
82
83 for (Iterator<String> iter = collected.iterator(); iter.hasNext();) {
84 filename = iter.next();
85 file = new File(filename);
86
87
88 fContainer = FieldContainer.createFieldContainer(getConfig());
89 fContainer.read(IOUtils.openReader(file, getConfig()));
90 predictions = new ExtractionContainer(
91 targetStruct, fContainer);
92
93
94 answerKeys = AnswerBuilder.readCorrespondingAnswerKeys(
95 targetStruct, file, getConfig());
96 evalContainer.evaluateBatch(predictions, answerKeys,
97 IOUtils.getBaseName(file));
98 }
99
100
101 final File outDir = IOUtils.determineOutputDirectory(getConfig());
102 final String baseName = "All";
103 final FieldContainer storage =
104 FieldContainer.createFieldContainer(getConfig());
105 evalContainer.storeEntries(storage);
106 final File extFile = IOUtils.createOutFile(outDir, baseName, "ext");
107 final Writer extWriter = IOUtils.openWriter(extFile, getConfig());
108 try {
109 storage.store(extWriter);
110 } finally {
111 IOUtils.tryToClose(extWriter);
112 }
113
114
115 final File metricsFile = IOUtils.createOutFile(outDir, baseName,
116 MultiFMetrics.EXT_METRICS);
117 final Writer metricsWriter = IOUtils.openWriter(metricsFile,
118 getConfig());
119
120 try {
121 final FieldContainer metricsStorage =
122 FieldContainer.createFieldContainer(getConfig());
123 evalContainer.viewMetrics().storeEntries(metricsStorage);
124 metricsStorage.store(metricsWriter);
125 metricsWriter.flush();
126 } finally {
127 IOUtils.tryToClose(metricsWriter);
128 }
129
130 Util.LOG.info("Stored evaluated extractions in " + extFile
131 + " metrics in " + metricsFile);
132 }
133
134 }