View Javadoc

1   /*
2    * Copyright (C) 2004-2006 Christian Siefkes <christian@siefkes.net>.
3    * Development of this software is supported by the German Research Society,
4    * Berlin-Brandenburg Graduate School in Distributed Information Systems
5    * (DFG grant no. GRK 316).
6    *
7    * This program is free software; you can redistribute it and/or modify
8    * it under the terms of the GNU General Public License as published by
9    * the Free Software Foundation; either version 2 of the License, or
10   * (at your option) any later version.
11   *
12   * This program is distributed in the hope that it will be useful,
13   * but WITHOUT ANY WARRANTY; without even the implied warranty of
14   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15   * GNU General Public License for more details.
16   *
17   * You should have received a copy of the GNU General Public License
18   * along with this program; if not, visit
19   * http://www.gnu.org/licenses/gpl.html or write to the Free Software
20   * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
21   */
22  package de.fu_berlin.ties.extract;
23  
24  import java.io.File;
25  import java.io.IOException;
26  import java.io.Reader;
27  import java.io.Writer;
28  import java.util.ArrayList;
29  import java.util.Iterator;
30  import java.util.LinkedList;
31  import java.util.List;
32  import java.util.NoSuchElementException;
33  import java.util.SortedMap;
34  import java.util.TreeMap;
35  
36  import org.apache.commons.lang.builder.ToStringBuilder;
37  import org.apache.commons.math.stat.descriptive.moment.StandardDeviation;
38  import org.dom4j.Document;
39  import org.dom4j.DocumentException;
40  
41  import de.fu_berlin.ties.Closeable;
42  import de.fu_berlin.ties.ContextMap;
43  import de.fu_berlin.ties.ParsingException;
44  import de.fu_berlin.ties.ProcessingException;
45  import de.fu_berlin.ties.TextProcessor;
46  import de.fu_berlin.ties.TiesConfiguration;
47  import de.fu_berlin.ties.classify.Prediction;
48  import de.fu_berlin.ties.classify.Tuner;
49  import de.fu_berlin.ties.eval.AccuracyView;
50  import de.fu_berlin.ties.eval.FMetrics;
51  import de.fu_berlin.ties.eval.FMetricsView;
52  import de.fu_berlin.ties.eval.MultiFMetrics;
53  import de.fu_berlin.ties.eval.MultiFMetricsView;
54  import de.fu_berlin.ties.io.FieldContainer;
55  import de.fu_berlin.ties.io.FieldMap;
56  import de.fu_berlin.ties.io.IOUtils;
57  import de.fu_berlin.ties.util.Util;
58  import de.fu_berlin.ties.xml.dom.DOMUtils;
59  
60  /***
61   * Trains an extractor and evaluates extraction quality. Processes shuffle
62   * files (as generated by {@link de.fu_berlin.ties.eval.ShuffleGenerator}
63   * contain the files to use for training and evaluation.
64   * For each of these files, a corresponding answer key (*.ans) must exist.
65   *
66   * <p>Instances of this class are not thread-safe.
67   *
68   * @author Christian Siefkes
69   * @version $Revision: 1.88 $, $Date: 2006/10/21 16:04:14 $, $Author: siefkes $
70   */
71  public class TrainEval extends TextProcessor implements Closeable {
72  
73      /***
74       * An inner class wrapping the results of a
75       * {@linkplain TrainEval#trainAndEval(String[], File, File, String, Writer)
76       * training + evaluation run}.
77       */
78      public static class Results {
79  
80          /***
81           * The map from TUNE iterations to
82           * {@link EvaluatedExtractionContainer}s containing the evaluated
83           * extractions (after the corresponding TUNE iteration).
84           */
85          private final SortedMap<Integer, EvaluatedExtractionContainer> evaluated
86              = new TreeMap<Integer, EvaluatedExtractionContainer>();
87  
88          /***
89           * A read-only view on the feature count statistics collected during
90           * training.
91           */
92  //        private FeatureCountView trainFeatureCV;
93  
94          /***
95           * A read-only view on the feature count statistics collected during
96           * extraction.
97           */
98  //        private FeatureCountView extractFeatureCV;
99  
100         /***
101          * Creates a new instance.
102          */
103         public Results() {
104             super();
105         }
106 
107         /***
108          * Adds an evaluated extraction container.
109          *
110          * @param iteration the number of the TUNE iteration to add
111          * @param evaluatedExtractions the container to add
112          * @throws IllegalArgumentException if there already is a container
113          * stored for <code>iteration</code>
114          */
115         private void addEvaluated(final int iteration,
116                 final EvaluatedExtractionContainer evaluatedExtractions) 
117         throws IllegalArgumentException {
118             final EvaluatedExtractionContainer oldValue =
119                 evaluated.put(Integer.valueOf(iteration), evaluatedExtractions);
120 
121             if (oldValue != null) {
122                 throw new IllegalArgumentException("Cannot store two "
123                         + "extraction containers for same iteration ("
124                         + iteration + "): " + oldValue + "; "
125                         + evaluatedExtractions);
126             }
127         }
128 
129         /***
130          * Returns one of the stored evaluated extraction containers.
131          *
132          * @param iteration the number of the TUNE iteration to return;
133          * must be contained in {@link #iterations()}, otherwise
134          * <code>null</code> will be returned
135          * @return the specified container
136          */
137         public EvaluatedExtractionContainer getEvaluated(
138                 final Integer iteration) {
139             return evaluated.get(iteration);
140         }
141 
142         /***
143          * Returns one of the stored evaluated extraction containers.
144          *
145          * @param iteration the number index of the TUNE iterations to return;
146          * must be contained in {@link #iterations()}, otherwise
147          * <code>null</code> will be returned
148          * @return the specified container
149          */
150         public EvaluatedExtractionContainer getEvaluated(final int iteration) {
151             return getEvaluated(Integer.valueOf(iteration));
152         }
153 
154         /***
155          * Returns a read-only view on the feature count statistics collected
156          * during training.
157          * @return the stored attribute
158          */
159 /*        public FeatureCountView getExtractFeatureCV() {
160             return extractFeatureCV;
161         } */
162 
163         /***
164          * Returns a read-only view on the feature count statistics collected
165          * during extraction.
166          * @return the stored attribute
167          */
168 /*        public FeatureCountView getTrainFeatureCV() {
169             return trainFeatureCV;
170         } */
171 
172         /***
173          * Returns an iterator over the TUNE iterations for which extraction
174          * results are stored.
175          *
176          * @return an iterator over the TUNE iterations
177          */
178         public Iterator<Integer> iterations() {
179             return evaluated.keySet().iterator();
180         }
181 
182         /***
183          * Stores a read-only view on the feature count statistics collected
184          * during training.
185          *
186          * @param extractFeatures a read-only view on the feature count
187          * statistics collected during extraction
188          */
189 /*        private void setExtractFeatureCV(
190                 final FeatureCountView extractFeatures) {
191             extractFeatureCV = extractFeatures;
192         } */
193 
194         /***
195          * Stores a read-only view on the feature count statistics collected
196          * during extraction.
197          *
198          * @param trainFeatures a read-only view on the feature count statistics
199          * collected during training
200          */
201 /*        private void setTrainFeatureCV(
202                 final FeatureCountView trainFeatures) {
203             trainFeatureCV = trainFeatures;
204         } */
205 
206         /***
207          * Returns a string representation of this object.
208          *
209          * @return a textual representation
210          */
211         public String toString() {
212             return new ToStringBuilder(this)
213 //                .append("training feature count", trainFeatureCV)
214 //                .append("extraction feature count", extractFeatureCV)
215                 .append("size", evaluated.size())
216                 .toString();
217         }
218 
219     }
220 
221     /***
222      * Configuration key: If <code>true</code>, a fully incremental setup is
223      * used where the trainer is trained on each document after the extractor
224      * processed it.
225      */
226     public static final String CONFIG_FEEDBACK = "eval.feedback";
227 
228     /***
229      * Configuration key: The maximum number of iterations used for TUNE
230      * training the sentence classifier; if 0 or negative, the value of
231      * {@link Tuner#CONFIG_TUNE} is used.
232      */
233     public static final String CONFIG_SENTENCE_TUNE = "sent.tune";
234 
235     /***
236      * Serialization key for the number of the iteration (when TUNE training).
237      */
238     public static final String KEY_ITERATION = "Iteration";
239 
240     /***
241      * Serialization key for the number of the run.
242      */
243     public static final String KEY_RUN = "Run";
244 
245     /***
246      * Serialization key for the type (either "Train" or "Eval").
247      */
248     public static final String KEY_TYPE = "Type";
249 
250     /***
251      * Serialization value for the "Train" type.
252      */
253     public static final String TYPE_TRAIN = "Train";
254 
255     /***
256      * Serialization value for the "Eval" type.
257      */
258     public static final String TYPE_EVAL = "Eval";
259 
260     /***
261      * File extension used for metrics files.
262      */
263     public static final String EXT_METRICS = "metrics";
264 
265     /***
266      * Results are measured after each 10th evaluated document if
267      * {@link #feedback} is given.
268      */
269     private static final int MEASURE_FEEDBACK = 10;
270 
271     /***
272      * If <code>true</code>, a fully incremental setup is used where the
273      * trainer is trained on each document after the extractor processed it.
274      */
275     private final boolean feedback;
276 
277     /***
278      * The maximum number of iterations used for TUNE training the sentence
279      * classifier; if 0 or negative, the value of
280      * {@link Tuner#getTuneIterations()} is used.
281      */
282     private final int sentenceIterations;
283 
284     /***
285      * The last TUNE iteration that was actually used for training in any
286      * batch of processed files.
287      */
288     private int lastUsedTuneIteration = -1;
289 
290     /***
291      * An array of multi-metrics calculating summaries esp. standard deviations
292      * over all runs. Each array element corresponds to one TUNE operation.
293      */
294     private final MultiFMetrics[] averages;
295 
296     /***
297      * An array of multi-metrics incrementally calculating summaries after
298      * evaluating each 10th document if {@link #feedback} is given.
299      */
300     private final List<MultiFMetrics> feedbackAverages;
301 
302     /***
303      * Calculate feature count statistics over all runs.
304      */
305 //    private final FieldContainer featureCounts;
306 
307     /***
308      * Stores precision and recall values for sentence filtering, if used.
309      */
310     private FieldContainer sentenceMetricsStore;
311 
312     /***
313      * The number of the current run.
314      */
315     private int runNo = 0;
316 
317     /***
318      * Used to control TUNE training.
319      */
320     private final Tuner tuner;
321 
322     /***
323      * Whether to evaluate predictions by comparing them to answer keys,
324      * otherwise predictions are stored without evaluating them.
325      */
326     private final boolean evaluate;
327 
328     /***
329      * The extension used to stored predictions (if {@link #evaluate} is set
330      * to <code>false</code>).
331      */
332     private final String predExtension;
333 
334     /***
335      * Whether to write prediction files to the configured output directory
336      * or the the directory containing the input file.
337      */
338     private final boolean storePredsInOutDir;
339 
340 
341     /***
342      * Creates a new instance, using a default extension and the
343      * {@linkplain TiesConfiguration#CONF standard configuration}.
344      *
345      * @throws IllegalArgumentException if the configured values are outside the
346      * allowed ranges
347      * @throws ClassCastException if the configured numeric values cannot be
348      * parsed
349      * @throws NoSuchElementException if one of the required values is
350      * missing from the configuration
351      */
352     public TrainEval() throws IllegalArgumentException, ClassCastException,
353     NoSuchElementException {
354         this(EXT_METRICS);
355     }
356 
357     /***
358      * Creates a new instance, using the {@linkplain TiesConfiguration#CONF
359      * standard configuration}.
360      *
361      * @param outExt the extension to use for output files
362      * @throws IllegalArgumentException if the configured values are outside the
363      * allowed ranges
364      * @throws ClassCastException if the configured numeric values cannot be
365      * parsed
366      * @throws NoSuchElementException if one of the required values is
367      * missing from the configuration
368      */
369     public TrainEval(final String outExt) throws IllegalArgumentException,
370             ClassCastException, NoSuchElementException {
371         this(outExt, TiesConfiguration.CONF);
372     }
373 
374     /***
375      * Creates a new instance.
376      *
377      * @param outExt the extension to use for output files
378      * @param config used to configure this instance
379      * @throws IllegalArgumentException if the configured values are outside the
380      * allowed ranges
381      * @throws ClassCastException if the configured numeric values cannot be
382      * parsed
383      * @throws NoSuchElementException if one of the required values is
384      * missing from the configuration
385      */
386     public TrainEval(final String outExt, final TiesConfiguration config)
387             throws IllegalArgumentException, ClassCastException,
388             NoSuchElementException {
389         this(outExt, new Tuner(config, ExtractorBase.CONFIG_SUFFIX_IE),
390                 config.getInt(CONFIG_SENTENCE_TUNE),
391                 config.getBoolean(CONFIG_FEEDBACK),
392                 config.getBoolean("extract.evaluate"),
393                 config.getString("extract.pred.ext"),
394                 config.getBoolean("extract.pred.use-outdir"), config);
395     }
396 
397     /***
398      * Creates a new instance.
399      *
400      * @param outExt the extension to use for output files
401      * @param theTuner used for TUNE training
402      * @param sentenceTUNE the maximum number of iterations used for TUNE
403      * training the sentence classifier (if used); if 0 or negative, the value
404      * of {@link Tuner#getTuneIterations()} is used
405      * @param giveFeedback if <code>true</code>, a fully incremental setup is
406      * used where the trainer is trained on each document after the extractor
407      * processed it; it's not allowed to set this both this and
408      * {@link Tuner#isTuneEach()} to <code>true</code> when training for
409      * several TUNE iterations because that would mean to evaluate on the
410      * training set
411      * @param doEvaluate whether to evaluate predictions by comparing them to
412      * answer keys, otherwise predictions are stored without evaluating them
413      * @param predExt the extension used to stored predictions (if
414      * <code>doEvaluate</code> is set to <code>false</code>)
415      * @param doStorePredsInOutDir whether to write prediction files to the
416      * configured output directory or the the directory containing the input
417      * file
418      * @param config used to configure superclasses, trainer, and extractor;
419      * if <code>null</code>, the {@linkplain TiesConfiguration#CONF standard
420      * configuration} is used
421      */
422     public TrainEval(final String outExt, final Tuner theTuner,
423             final int sentenceTUNE, final boolean giveFeedback,
424             final boolean doEvaluate, final String predExt,
425             final boolean doStorePredsInOutDir,
426             final TiesConfiguration config) {
427         super(outExt, config);
428 
429         tuner = theTuner;
430         sentenceIterations = sentenceTUNE;
431         feedback = giveFeedback;
432         averages = new MultiFMetrics[tuner.getTuneIterations()];
433 //        featureCounts = FieldContainer.createFieldContainer(config);
434         evaluate = doEvaluate;
435         predExtension = predExt;
436         storePredsInOutDir = doStorePredsInOutDir;
437 
438         if (giveFeedback && (tuner.getTuneIterations() > 1)
439                 && tuner.isTuneEach()) {
440             throw new IllegalArgumentException(
441                     "It's not allowed give feedback "
442                     + "when evaluating after each of several iteration because "
443                     + "that would mean to evaluate on the training set");
444         }
445 
446         // init metrics to store incremental F1 values if feedback is given
447         if (feedback) {
448             feedbackAverages = new ArrayList<MultiFMetrics>();
449         } else {
450             feedbackAverages = null;
451         }
452 
453         // Dummy serialization to preload statistical implementations.
454         // This avoid instantiation error in close() method if the JAR file
455         // has changed in the meanwhile (which might happen if a nightly build
456         // takes place during a long test run).
457         final FieldContainer dummy = new FieldContainer();
458         final MultiFMetrics dummyMetrics = new MultiFMetrics(true);
459         dummyMetrics.storeEntries(dummy);
460         StandardDeviation dummyDev = new StandardDeviation();
461         dummyDev.clear();
462     }
463 
464     /***
465      * Check whether to disable sentence training; called after each training
466      * iteration.
467      * 
468      * @param trainer the used trainer
469      * @param sentenceMetrics metrics calculated for sentence filtering, if
470      * used (<code>null</code> otherwise)
471      * @param iteration the current iteration
472      */
473     private void checkSentenceTraining(final Trainer trainer,
474             final FMetrics sentenceMetrics, final int iteration) {
475         if ((sentenceIterations > 0) && (sentenceIterations == iteration)) {
476             trainer.disableSentenceTraining();
477             Util.LOG.info("Disabled sentence training after " + iteration
478                     + " iterations");
479         }
480 
481         if (sentenceMetrics != null) {
482             updateSentenceMetricsStore(sentenceMetrics, true, iteration);
483         }
484     }
485 
486     /***
487      * {@inheritDoc}
488      */
489     public void close(final int errorCount)
490             throws IOException, ProcessingException {
491         // don't do anything if there were errors
492         if (errorCount <= 0) {
493             final File outDir = IOUtils.determineOutputDirectory(getConfig());
494 
495             // store feature counts + average metrics incl. standard deviations
496 //            storeValues(featureCounts, outDir, "FeatureCounts", null);
497 
498             final FieldContainer averagesStore =
499                 FieldContainer.createFieldContainer(getConfig());
500             boolean storeIterations = (averages.length > 1);
501 
502             // don't store results for more TUNE iterations than actually used
503             final int lastToStore = Math.min(averages.length,
504                     lastUsedTuneIteration);
505 
506             for (int i = 1; i <= lastToStore; i++) {
507                 // Some might be null because of eval.tune.since or ...list.
508                 // If tuneEach is disabled, we only store iterations from the
509                 // tuneEvaluations list + the very last one.
510                 if (averages[i - 1] != null && (tuner.isTuneEach()
511                         || tuner.getTuneEvaluations().contains(
512                                 Integer.valueOf(i)) || i == lastToStore)) {
513                     // add iteration number if we TUNEd
514                     if (storeIterations) {
515                         averagesStore.backgroundMap().put(KEY_ITERATION,
516                                 Integer.valueOf(i));
517                     }
518 
519                     averages[i - 1].storeEntries(averagesStore);
520                 }
521             }
522             averagesStore.storeInFile(outDir, "All", MultiFMetrics.EXT_METRICS,
523                     getConfig());
524 
525             // store incremental F1 if feedback is enabled
526             if (feedbackAverages != null) {
527                 final FieldContainer feedbackAveragesStore =
528                     FieldContainer.createFieldContainer(getConfig());
529 
530                 for (int i = 0; i < feedbackAverages.size(); i++) {
531                     feedbackAverages.get(i).storeEntries(feedbackAveragesStore);
532                 }
533                 feedbackAveragesStore.storeInFile(outDir, "Feedback",
534                         MultiFMetrics.EXT_METRICS, getConfig());
535             }
536         }
537     }
538 
539     /***
540      * {@inheritDoc}
541      */
542     protected void doProcess(final Reader reader, final Writer writer,
543             final ContextMap context) throws IOException, ProcessingException {
544         // increase run number + read list of files
545         runNo++;
546         final String[] files = IOUtils.readURIList(reader);
547 
548         if (files.length == 0) {
549             Util.LOG.info("No files to process");
550         } else {
551             final String baseName = IOUtils.getBaseName(
552                 new File((String) context.get(KEY_LOCAL_NAME)));
553             final File inDir = (File) context.get(KEY_DIRECTORY);
554             final File outDir = IOUtils.determineOutputDirectory(getConfig());
555             Results currentResults;
556 //            FieldMap currentFMap;
557 
558             MultiFMetricsView currentMetrics = null;
559 
560             //Util.LOG.info("Starting evaluation run " + runNo);
561             currentResults =
562                 trainAndEval(files, inDir, outDir, baseName, writer);
563 
564             if (currentResults != null) {
565 /*                // store feature counts for training + evaluation
566                 final Integer runNumber = new Integer(runNo);
567                 currentFMap = currentResults.getTrainFeatureCV().storeFields();
568                 currentFMap.put(KEY_RUN, runNumber);
569                 currentFMap.put(MultiFMetrics.KEY_TYPE, "training");
570                 featureCounts.add(currentFMap);
571 
572                 currentFMap =
573                     currentResults.getExtractFeatureCV().storeFields();
574                 currentFMap.put(KEY_RUN, runNumber);
575                 currentFMap.put(MultiFMetrics.KEY_TYPE, "extraction");
576                 featureCounts.add(currentFMap); */
577 
578                 // update average metrics
579                 final Iterator<Integer> iterIter = currentResults.iterations();
580                 Integer iteration;
581                 int it = 0;
582 
583                 while (iterIter.hasNext()) {
584                     iteration = iterIter.next();
585                     it = iteration.intValue();
586                     currentMetrics =
587                         currentResults.getEvaluated(iteration).viewMetrics();
588                     if (averages[it-1] == null) {
589                         averages[it-1] = new MultiFMetrics(true);
590                     }
591                     averages[it-1].update(currentMetrics);
592                 }
593 
594                 // if there are additional averages it means that we stopped
595                 // TUNEing early, so the last result is valid for all later 
596                 // iterations too
597                 for (it++; it <= averages.length; it++) {
598                     if (averages[it-1] == null) {
599                         averages[it-1] = new MultiFMetrics(true);
600                     }
601 
602                     // this will copy the last existing result which is what
603                     // we want
604                     averages[it-1].update(currentMetrics);
605                 }
606             }
607         }
608     }
609 
610     /***
611      * Creates and initializes a extractor to use for an evaluation run,
612      * re-using the components of the provided trainer. Subclasses can
613      * overwrite this method to provide a different extractor.
614      *
615      * @param trainer trainer whose components should be re-used
616      * @return the created extractor
617      */
618     protected Extractor initExtractor(final Trainer trainer) {
619         // we don't need an output extension
620         return new Extractor(null, trainer);
621     }
622 
623     /***
624      * Creates and initializes a trainer to use for an evaluation run,
625      * configured from the
626      * {@link de.fu_berlin.ties.ConfigurableProcessor#getConfig() stored
627      * configuration}. Subclasses can overwrite this method to provide a
628      * different trainer.
629      *
630      * @param runDirectory directory used to run the classifier
631      * @return the created trainer
632      * @throws ProcessingException if an error occurs during initialization
633      */
634     protected Trainer initTrainer(final File runDirectory)
635             throws ProcessingException {
636         // we don't need an output extension
637         final Trainer trainer = new Trainer(null, runDirectory, getConfig());
638         // ensure the prediction model is empty
639         trainer.reset();
640         return trainer;
641     }
642 
643     /***
644      * Returns a string representation of this object.
645      *
646      * @return a textual representation
647      */
648     public String toString() {
649         return new ToStringBuilder(this)
650             .appendSuper(super.toString())
651             .append("tuner", tuner)
652             .append("sentence iterations", sentenceIterations)
653             .append("feedback", feedback)
654             .toString();
655     }
656 
657     /***
658      * Helper method for serializing the calculated metrics.
659      *
660      * @param outDirectory directory used to do this run and store the results
661      * @param baseName the base name of the files to use for storing
662      * all extractions and training statistics
663      * @param iteration the number of the current iteration
664      * @param evaluated the container of evaluated extractions to tore
665      * @param sentenceMetrics metrics calculated for sentence filtering, if
666      * used (<code>null</code> otherwise)
667      * @throws IOException if an I/O error occurs while serializing
668      */
669     private void serializeExtractions(final File outDirectory,
670             final String baseName, final int iteration,
671             final EvaluatedExtractionContainer evaluated,
672             final FMetrics sentenceMetrics)
673     throws IOException {
674         // serialize + store evaluated extractions
675         final FieldContainer resultStorage =
676             FieldContainer.createFieldContainer(getConfig());
677         evaluated.storeEntries(resultStorage);
678         final File evaluatedFile = resultStorage.storeInFile(outDirectory,
679                 baseName, Extractor.EXT_EXTRACTIONS, getConfig());
680         Util.LOG.info("Stored results of training + evaluation run in "
681             + evaluatedFile);
682 
683         // update metrics of sentence filtering, if used
684         if (sentenceMetrics != null) {
685             updateSentenceMetricsStore(sentenceMetrics, false, iteration);
686         }
687     }
688 
689     /***
690      * Helper method for serializing the calculated metrics.
691      *
692      * @param outDirectory directory used to do this run and store the results
693      * @param baseName the base name of the files to use for storing
694      * all extractions and training statistics
695      * @param writer used to serialize the calculated metrics
696      * @param results contains the evaluated extractions
697      * @throws IOException if an I/O error occurs while serializing the metrics
698      */
699     private void serializeMetrics(final File outDirectory,
700             final String baseName, final Writer writer,
701             final Results results)
702     throws IOException {
703         // serialize metrics to output file
704         final FieldContainer metricsStore =
705             FieldContainer.createFieldContainer(getConfig());
706         Integer iteration;
707         EvaluatedExtractionContainer evaluated;
708         final boolean storeIterations = (tuner.getTuneIterations() > 1);
709         Iterator<Integer> tuneIter = results.iterations();
710 
711         while (tuneIter.hasNext()) {
712             iteration = tuneIter.next();
713             evaluated = results.getEvaluated(iteration);
714 
715             // add iteration number, if there are several ones to store
716             if (storeIterations) {
717                 metricsStore.backgroundMap().put(KEY_ITERATION, iteration);
718             }
719 
720             evaluated.viewMetrics().storeEntries(metricsStore);
721         }
722 
723         metricsStore.store(writer);
724         writer.flush();
725 
726         // serialize sentence metrics
727         if (sentenceMetricsStore != null) {
728             sentenceMetricsStore.storeInFile(outDirectory, baseName, "sent",
729                     getConfig());
730         }
731     }
732 
733     /***
734      * Helper method for serializing the training accuracies.
735      *
736      * @param outDirectory directory used to do this run and store the results
737      * @param baseName the base name of the files to use for storing
738      * all extractions and training statistics
739      * @param accContainers a an of containers used to store the accuracy of
740      * each classifier, might be <code>null</code> if accuracies aren't measured
741      * @param globalAcc an array of accuracies, one for each classifier
742      * @throws IOException if an I/O error occurs while serializing
743      */
744     private void serializeTrainingMetrics(final File outDirectory,
745             final String baseName, final FieldContainer[] accContainers,
746             final AccuracyView[] globalAcc)
747     throws IOException {
748         if (accContainers != null) {
749             String name;
750 
751             for (int j = 0; j < globalAcc.length; j++) {
752                 if (globalAcc.length > 1) {
753                     // append prefix letter (a, b, c etc.)
754                     name = baseName + (char) ('a' + j);
755                 } else {
756                     // sufficient to use base name
757                     name = baseName;
758                 }
759                 accContainers[j].storeInFile(outDirectory, name, "train",
760                         getConfig());
761             }
762         }
763     }
764 
765     /***
766      * Processes an array of files. For each file, a corresponding answer key
767      * (*.ans) must exist.
768      *
769      * @param files the array of file names to process (relative to the
770      * <code>inDirectory</code>)
771      * @param inDirectory directory containing the files to process
772      * @param outDirectory directory used to do this run and store the results
773      * @param baseName the base name of the files to use for storing
774      * all extractions and training statistics
775      * @param writer used to serialize the calculated metrics
776      * @return a wrapper of the results of this run
777      * @throws IOException if an I/O error occurs
778      * @throws ProcessingException if an error occurs during processing
779      */
780     public Results trainAndEval(final String[] files, final File inDirectory,
781             final File outDirectory, final String baseName,
782             final Writer writer) throws IOException, ProcessingException {
783         // make given directory default dir
784         IOUtils.setDefaultDirectory(outDirectory);
785 
786         // select files for training + evaluation
787         final List<String> trainFiles = new LinkedList<String>();
788         final List<String> evalFiles = new LinkedList<String>();
789         tuner.selectFiles(files, trainFiles, evalFiles);
790 
791         final long trainStartTime = System.currentTimeMillis();
792         File currentFile;
793         Document currentDoc;
794         ExtractionContainer currentAnswers;
795 
796         // store token accuracies during TOE training
797         FieldContainer[] accContainers = null;
798         AccuracyView[] currentAcc;
799         AccuracyView[] globalAcc = null;
800         FieldMap accMap;
801         final boolean storeIterations = (tuner.getTuneIterations() > 1);
802 
803         // measure accuracies for TUNE training
804         boolean continueTraining = true;
805         double[] currentOverallAcc;
806         int evalFileNum, feedbackNum;
807 
808         // reset TUNER state
809         tuner.reset();
810 
811         // create new trainer
812         final Trainer trainer = initTrainer(outDirectory);
813         Extractor extractor = null;
814 
815         // measure sentence filtering, if used
816         FMetrics sentenceMetrics = null;
817         FMetricsView newSentenceMetrics;
818 
819         if (trainer.isSentenceFiltering()) {
820             sentenceMetricsStore =
821                 FieldContainer.createFieldContainer(getConfig());
822         } else {
823             sentenceMetricsStore = null;
824         }
825 
826         // store evaluated extraction containers, if evaluting
827         final Results results = (evaluate) ? new Results() : null;
828 
829 /*        // pseudo-randonness generator if TUNEing
830         final Random pseudoRandom = (tuneIterations > 1) ?
831                 Util.reproducibleRandom() : null; */
832 
833         for (int i = 1; (i <= tuner.getTuneIterations())
834                 && continueTraining; i++) {
835             if (tuner.getTuneIterations() > 1) {
836                 Util.LOG.debug("Starting TUNE iteration " + i + "/" 
837                         + tuner.getTuneIterations()
838                         + " (will stop if no improvement for "
839                         + tuner.getTuneStop() + " iterations)");
840             }
841 
842 /* degrades results:
843             // reshuffling train files in 2nd + following TUNE iterations
844             if (i > 1) {
845                 Collections.shuffle(trainFiles, pseudoRandom);
846                 Util.LOG.debug("Pseudo-randomly reshuffled training files");
847             } */
848 
849             final Iterator trainIter = trainFiles.iterator();
850             trainer.resetGlobalAccuracy();
851 
852             // measure sentence filtering, if used
853             if (trainer.isSentenceFiltering()) {
854                 sentenceMetrics = new FMetrics();
855             }
856 
857             //  run on trainFiles
858             while (trainIter.hasNext()) {
859                 currentFile = IOUtils.resolveFilename(
860                         inDirectory, (String) trainIter.next());
861                 Util.LOG.debug("Starting to train " + currentFile);
862 
863                 try {
864                     currentDoc =
865                         DOMUtils.readDocument(currentFile, getConfig());
866                 } catch (DocumentException de) {
867                     // wrap exception
868                     throw new ParsingException("Error while parsing "
869                             + currentFile + ": " + de.toString(), de);
870                 }
871 
872                 // read answer keys in DSV format (*.ans file must exist)
873                 currentAnswers = AnswerBuilder.readCorrespondingAnswerKeys(
874                     trainer.getTargetStructure(), currentFile, getConfig());
875 
876                 // train the trainer
877                 currentAcc =
878                     trainer.train(currentDoc, currentFile, currentAnswers);
879 
880                 // store global + local accuracy if in TOE mode
881                 if (currentAcc != null) {
882                     globalAcc = trainer.viewGlobalAccuracy();
883 
884                     // initialize containers once
885                     if (accContainers == null) {
886                         accContainers = new FieldContainer[globalAcc.length];
887                         for (int j = 0; j < globalAcc.length; j++) {
888                             accContainers[j] =
889                                 FieldContainer.createFieldContainer(
890                                         getConfig());
891                         }
892                     }
893 
894                     // store each accuracy
895                     for (int j = 0; j < globalAcc.length; j++) {
896                         accMap = globalAcc[j].storeFields();
897                         accMap.putAll(currentAcc[j].storeFields());
898                         accMap.put(Prediction.KEY_SOURCE,
899                             IOUtils.getBaseName(currentFile));
900                         if (storeIterations) {
901                             accMap.put(KEY_ITERATION, new Integer(i));
902                         }
903                         accContainers[j].add(accMap);
904                     }
905                 }
906                 Util.LOG.info("Trained " + currentFile);
907 
908                 // measure sentence filtering, if used
909                 if (trainer.isSentenceFiltering()) {
910                     newSentenceMetrics = trainer.evaluateSentenceFiltering();
911                     Util.LOG.debug("Evaluated sentence filtering for trained "
912                             + "document: " + newSentenceMetrics);
913                     sentenceMetrics.update(newSentenceMetrics);
914                 }
915             }
916 
917             // check whether to continue sentence training + update metrics
918             checkSentenceTraining(trainer, sentenceMetrics, i);
919 
920             // stop TUNE if all accuracies did not or cannot longer increase
921             if (globalAcc != null) {
922                 currentOverallAcc = new double[globalAcc.length];
923 
924                 for (int j = 0; j < currentOverallAcc.length; j++) {
925                     currentOverallAcc[j] = globalAcc[j].getAccuracy();
926                 }
927 
928                 continueTraining = tuner.continueTraining(currentOverallAcc, i);
929             }
930 
931             if (tuner.shouldEvaluate(continueTraining, i)) {
932                 // evaluate extractions
933                 Util.LOG.info("Finished training using " + trainer.toString()
934                         + "; " + Util.showDuration(trainStartTime));
935                 final long evalStartTime = System.currentTimeMillis();
936                 evalFileNum = 0;
937 
938                 // create extractor (first time only)
939                 if (extractor == null) {
940                     extractor = initExtractor(trainer);
941                 }
942 
943                 final Iterator evalIter = evalFiles.iterator();
944                 final EvaluatedExtractionContainer evaluated = (evaluate)
945                     ? new EvaluatedExtractionContainer(
946                         extractor.getTargetStructure(), getConfig())
947                     : null;
948                 ExtractionContainer currentResults;
949                 MultiFMetricsView interimMetrics;
950 
951                 // measure sentence filtering, if used
952                 if (extractor.isSentenceFiltering()) {
953                     sentenceMetrics = new FMetrics();
954                  }
955 
956                 // run on evalFiles
957                 while (evalIter.hasNext()) {
958                     currentFile = IOUtils.resolveFilename(inDirectory,
959                         (String) evalIter.next());
960                     evalFileNum++;
961                     Util.LOG.debug("Starting to extract and evaluate file #"
962                             + evalFileNum + ": " + currentFile);
963 
964                     try {
965                         currentDoc = DOMUtils.readDocument(currentFile,
966                             getConfig());
967                     } catch (DocumentException de) {
968                         // wrap exception
969                         throw new ParsingException(de);
970                     }
971 
972                     // extract results
973                     currentResults = extractor.extract(currentDoc, currentFile);
974 
975                     if (evaluate) {
976                         // read answer key
977                         currentAnswers =
978                             AnswerBuilder.readCorrespondingAnswerKeys(
979                                     trainer.getTargetStructure(), currentFile,
980                                     getConfig());
981 
982                         // invoke the trainer if feedback should be given
983                         if (feedback) {
984                             currentAcc = trainer.train(currentDoc, currentFile,
985                                     currentAnswers);
986                         }
987 
988                         // measure sentence filtering, if used (we do this prior
989                         // to evaluating the predicted extractions because
990                         // evaluateBatch destructively modifies the answer keys)
991                         if (extractor.isSentenceFiltering()) {
992                             newSentenceMetrics =
993                                 extractor.evaluateSentenceFiltering(
994                                         currentAnswers);
995                             Util.LOG.debug("Evaluated sentence filtering for "
996                                 + "current document: " + newSentenceMetrics);
997                             sentenceMetrics.update(newSentenceMetrics);
998                         }
999 
1000                         // compare with results, storing base name as key
1001                         evaluated.evaluateBatch(currentResults, currentAnswers,
1002                             IOUtils.getBaseName(currentFile));
1003 
1004                         // log results
1005                         interimMetrics = evaluated.viewMetrics();
1006                         Util.LOG.info("Extracted and evaluated " + currentFile
1007                             + ", interim results: " + interimMetrics.viewAll());
1008 
1009                         // store results after each 10th file (and after last)
1010                         // if feedback gis iven
1011                         if (feedback && ((evalFileNum % MEASURE_FEEDBACK == 0)
1012                                 || !evalIter.hasNext())) {
1013                             feedbackNum = new Double(
1014                                     Math.ceil((double) evalFileNum
1015                                     / MEASURE_FEEDBACK)).intValue();
1016 
1017                             // initialize new metrics if necessary (first run)
1018                             if (feedbackAverages.size() < feedbackNum) {
1019                                 feedbackAverages.add(new MultiFMetrics(true));
1020                             }
1021 
1022                             // add current results
1023                             feedbackAverages.get(feedbackNum - 1).update(
1024                                     evaluated.viewMetrics());
1025                         }
1026                     } else {
1027                         // store current results
1028                         final File predDir = storePredsInOutDir
1029                             ? (outDirectory) : currentFile.getParentFile();
1030                         final String localName = IOUtils.getBaseName(
1031                                 currentFile) + '.' + predExtension;
1032                         final File predFile = new File(predDir, localName);
1033                         final FieldContainer predStorage =
1034                             FieldContainer.createFieldContainer(getConfig());
1035                         currentResults.storeEntries(predStorage);
1036                         final Writer predWriter =
1037                             IOUtils.openWriter(predFile, getConfig());
1038                         predStorage.store(predWriter);
1039                         predWriter.flush();
1040                         IOUtils.tryToClose(predWriter);
1041                         Util.LOG.info("Stored predicted extractions in "
1042                                 + predFile);
1043                     }
1044                 } // while evalIter
1045 
1046                 if (evaluate) {
1047                     serializeExtractions(outDirectory, baseName, i, evaluated,
1048                             sentenceMetrics);
1049                     results.addEvaluated(i, evaluated);
1050                 }
1051                 Util.LOG.info("Finished extraction and evaluation using "
1052                         + extractor.toString() + "; "
1053                         + Util.showDuration(evalStartTime));
1054             }
1055 
1056             // update max. number of iterations used in any batch if higher
1057             lastUsedTuneIteration = Math.max(lastUsedTuneIteration, i);
1058         }
1059 
1060         // serialize metrics + destroy extractor + return extractions
1061         serializeTrainingMetrics(outDirectory, baseName, accContainers,
1062                 globalAcc);
1063 
1064         if (evaluate) {
1065             serializeMetrics(outDirectory, baseName, writer, results);
1066 /*            results.setTrainFeatureCV(trainer.viewFeatureCount());
1067             results.setExtractFeatureCV(extractor.viewFeatureCount()); */
1068         }
1069 
1070         extractor.destroy();
1071         return results;
1072     }
1073 
1074     /***
1075      * Adds the results of a training or evaluation iteration to the statistics
1076      * calculated for sentence filtering.
1077      *
1078      * @param newMetrics the metrics to add
1079      * @param isTraining whether this is a training or an evaluation result
1080      * @param iteration the number of the iteration (counting from 1)
1081      */
1082     private void updateSentenceMetricsStore(final FMetricsView newMetrics,
1083             final boolean isTraining, final int iteration) {
1084         final FieldMap fields = newMetrics.storeFields();
1085 
1086         // type is either "Train" or "Eval"
1087         fields.put(KEY_TYPE, isTraining ? TYPE_TRAIN : TYPE_EVAL);
1088         fields.put(KEY_ITERATION, new Integer(iteration));
1089         sentenceMetricsStore.add(fields);
1090     }
1091 
1092 }