1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 package de.fu_berlin.ties.extract;
23
24 import java.io.File;
25 import java.io.IOException;
26 import java.io.Reader;
27 import java.io.Writer;
28 import java.util.ArrayList;
29 import java.util.Iterator;
30 import java.util.LinkedList;
31 import java.util.List;
32 import java.util.NoSuchElementException;
33 import java.util.SortedMap;
34 import java.util.TreeMap;
35
36 import org.apache.commons.lang.builder.ToStringBuilder;
37 import org.apache.commons.math.stat.descriptive.moment.StandardDeviation;
38 import org.dom4j.Document;
39 import org.dom4j.DocumentException;
40
41 import de.fu_berlin.ties.Closeable;
42 import de.fu_berlin.ties.ContextMap;
43 import de.fu_berlin.ties.ParsingException;
44 import de.fu_berlin.ties.ProcessingException;
45 import de.fu_berlin.ties.TextProcessor;
46 import de.fu_berlin.ties.TiesConfiguration;
47 import de.fu_berlin.ties.classify.Prediction;
48 import de.fu_berlin.ties.classify.Tuner;
49 import de.fu_berlin.ties.eval.AccuracyView;
50 import de.fu_berlin.ties.eval.FMetrics;
51 import de.fu_berlin.ties.eval.FMetricsView;
52 import de.fu_berlin.ties.eval.MultiFMetrics;
53 import de.fu_berlin.ties.eval.MultiFMetricsView;
54 import de.fu_berlin.ties.io.FieldContainer;
55 import de.fu_berlin.ties.io.FieldMap;
56 import de.fu_berlin.ties.io.IOUtils;
57 import de.fu_berlin.ties.util.Util;
58 import de.fu_berlin.ties.xml.dom.DOMUtils;
59
60 /***
61 * Trains an extractor and evaluates extraction quality. Processes shuffle
62 * files (as generated by {@link de.fu_berlin.ties.eval.ShuffleGenerator}
63 * contain the files to use for training and evaluation.
64 * For each of these files, a corresponding answer key (*.ans) must exist.
65 *
66 * <p>Instances of this class are not thread-safe.
67 *
68 * @author Christian Siefkes
69 * @version $Revision: 1.88 $, $Date: 2006/10/21 16:04:14 $, $Author: siefkes $
70 */
71 public class TrainEval extends TextProcessor implements Closeable {
72
73 /***
74 * An inner class wrapping the results of a
75 * {@linkplain TrainEval#trainAndEval(String[], File, File, String, Writer)
76 * training + evaluation run}.
77 */
78 public static class Results {
79
80 /***
81 * The map from TUNE iterations to
82 * {@link EvaluatedExtractionContainer}s containing the evaluated
83 * extractions (after the corresponding TUNE iteration).
84 */
85 private final SortedMap<Integer, EvaluatedExtractionContainer> evaluated
86 = new TreeMap<Integer, EvaluatedExtractionContainer>();
87
88 /***
89 * A read-only view on the feature count statistics collected during
90 * training.
91 */
92
93
94 /***
95 * A read-only view on the feature count statistics collected during
96 * extraction.
97 */
98
99
100 /***
101 * Creates a new instance.
102 */
103 public Results() {
104 super();
105 }
106
107 /***
108 * Adds an evaluated extraction container.
109 *
110 * @param iteration the number of the TUNE iteration to add
111 * @param evaluatedExtractions the container to add
112 * @throws IllegalArgumentException if there already is a container
113 * stored for <code>iteration</code>
114 */
115 private void addEvaluated(final int iteration,
116 final EvaluatedExtractionContainer evaluatedExtractions)
117 throws IllegalArgumentException {
118 final EvaluatedExtractionContainer oldValue =
119 evaluated.put(Integer.valueOf(iteration), evaluatedExtractions);
120
121 if (oldValue != null) {
122 throw new IllegalArgumentException("Cannot store two "
123 + "extraction containers for same iteration ("
124 + iteration + "): " + oldValue + "; "
125 + evaluatedExtractions);
126 }
127 }
128
129 /***
130 * Returns one of the stored evaluated extraction containers.
131 *
132 * @param iteration the number of the TUNE iteration to return;
133 * must be contained in {@link #iterations()}, otherwise
134 * <code>null</code> will be returned
135 * @return the specified container
136 */
137 public EvaluatedExtractionContainer getEvaluated(
138 final Integer iteration) {
139 return evaluated.get(iteration);
140 }
141
142 /***
143 * Returns one of the stored evaluated extraction containers.
144 *
145 * @param iteration the number index of the TUNE iterations to return;
146 * must be contained in {@link #iterations()}, otherwise
147 * <code>null</code> will be returned
148 * @return the specified container
149 */
150 public EvaluatedExtractionContainer getEvaluated(final int iteration) {
151 return getEvaluated(Integer.valueOf(iteration));
152 }
153
154 /***
155 * Returns a read-only view on the feature count statistics collected
156 * during training.
157 * @return the stored attribute
158 */
159
160
161
162
163 /***
164 * Returns a read-only view on the feature count statistics collected
165 * during extraction.
166 * @return the stored attribute
167 */
168
169
170
171
172 /***
173 * Returns an iterator over the TUNE iterations for which extraction
174 * results are stored.
175 *
176 * @return an iterator over the TUNE iterations
177 */
178 public Iterator<Integer> iterations() {
179 return evaluated.keySet().iterator();
180 }
181
182 /***
183 * Stores a read-only view on the feature count statistics collected
184 * during training.
185 *
186 * @param extractFeatures a read-only view on the feature count
187 * statistics collected during extraction
188 */
189
190
191
192
193
194 /***
195 * Stores a read-only view on the feature count statistics collected
196 * during extraction.
197 *
198 * @param trainFeatures a read-only view on the feature count statistics
199 * collected during training
200 */
201
202
203
204
205
206 /***
207 * Returns a string representation of this object.
208 *
209 * @return a textual representation
210 */
211 public String toString() {
212 return new ToStringBuilder(this)
213
214
215 .append("size", evaluated.size())
216 .toString();
217 }
218
219 }
220
221 /***
222 * Configuration key: If <code>true</code>, a fully incremental setup is
223 * used where the trainer is trained on each document after the extractor
224 * processed it.
225 */
226 public static final String CONFIG_FEEDBACK = "eval.feedback";
227
228 /***
229 * Configuration key: The maximum number of iterations used for TUNE
230 * training the sentence classifier; if 0 or negative, the value of
231 * {@link Tuner#CONFIG_TUNE} is used.
232 */
233 public static final String CONFIG_SENTENCE_TUNE = "sent.tune";
234
235 /***
236 * Serialization key for the number of the iteration (when TUNE training).
237 */
238 public static final String KEY_ITERATION = "Iteration";
239
240 /***
241 * Serialization key for the number of the run.
242 */
243 public static final String KEY_RUN = "Run";
244
245 /***
246 * Serialization key for the type (either "Train" or "Eval").
247 */
248 public static final String KEY_TYPE = "Type";
249
250 /***
251 * Serialization value for the "Train" type.
252 */
253 public static final String TYPE_TRAIN = "Train";
254
255 /***
256 * Serialization value for the "Eval" type.
257 */
258 public static final String TYPE_EVAL = "Eval";
259
260 /***
261 * File extension used for metrics files.
262 */
263 public static final String EXT_METRICS = "metrics";
264
265 /***
266 * Results are measured after each 10th evaluated document if
267 * {@link #feedback} is given.
268 */
269 private static final int MEASURE_FEEDBACK = 10;
270
271 /***
272 * If <code>true</code>, a fully incremental setup is used where the
273 * trainer is trained on each document after the extractor processed it.
274 */
275 private final boolean feedback;
276
277 /***
278 * The maximum number of iterations used for TUNE training the sentence
279 * classifier; if 0 or negative, the value of
280 * {@link Tuner#getTuneIterations()} is used.
281 */
282 private final int sentenceIterations;
283
284 /***
285 * The last TUNE iteration that was actually used for training in any
286 * batch of processed files.
287 */
288 private int lastUsedTuneIteration = -1;
289
290 /***
291 * An array of multi-metrics calculating summaries esp. standard deviations
292 * over all runs. Each array element corresponds to one TUNE operation.
293 */
294 private final MultiFMetrics[] averages;
295
296 /***
297 * An array of multi-metrics incrementally calculating summaries after
298 * evaluating each 10th document if {@link #feedback} is given.
299 */
300 private final List<MultiFMetrics> feedbackAverages;
301
302 /***
303 * Calculate feature count statistics over all runs.
304 */
305
306
307 /***
308 * Stores precision and recall values for sentence filtering, if used.
309 */
310 private FieldContainer sentenceMetricsStore;
311
312 /***
313 * The number of the current run.
314 */
315 private int runNo = 0;
316
317 /***
318 * Used to control TUNE training.
319 */
320 private final Tuner tuner;
321
322 /***
323 * Whether to evaluate predictions by comparing them to answer keys,
324 * otherwise predictions are stored without evaluating them.
325 */
326 private final boolean evaluate;
327
328 /***
329 * The extension used to stored predictions (if {@link #evaluate} is set
330 * to <code>false</code>).
331 */
332 private final String predExtension;
333
334 /***
335 * Whether to write prediction files to the configured output directory
336 * or the the directory containing the input file.
337 */
338 private final boolean storePredsInOutDir;
339
340
341 /***
342 * Creates a new instance, using a default extension and the
343 * {@linkplain TiesConfiguration#CONF standard configuration}.
344 *
345 * @throws IllegalArgumentException if the configured values are outside the
346 * allowed ranges
347 * @throws ClassCastException if the configured numeric values cannot be
348 * parsed
349 * @throws NoSuchElementException if one of the required values is
350 * missing from the configuration
351 */
352 public TrainEval() throws IllegalArgumentException, ClassCastException,
353 NoSuchElementException {
354 this(EXT_METRICS);
355 }
356
357 /***
358 * Creates a new instance, using the {@linkplain TiesConfiguration#CONF
359 * standard configuration}.
360 *
361 * @param outExt the extension to use for output files
362 * @throws IllegalArgumentException if the configured values are outside the
363 * allowed ranges
364 * @throws ClassCastException if the configured numeric values cannot be
365 * parsed
366 * @throws NoSuchElementException if one of the required values is
367 * missing from the configuration
368 */
369 public TrainEval(final String outExt) throws IllegalArgumentException,
370 ClassCastException, NoSuchElementException {
371 this(outExt, TiesConfiguration.CONF);
372 }
373
374 /***
375 * Creates a new instance.
376 *
377 * @param outExt the extension to use for output files
378 * @param config used to configure this instance
379 * @throws IllegalArgumentException if the configured values are outside the
380 * allowed ranges
381 * @throws ClassCastException if the configured numeric values cannot be
382 * parsed
383 * @throws NoSuchElementException if one of the required values is
384 * missing from the configuration
385 */
386 public TrainEval(final String outExt, final TiesConfiguration config)
387 throws IllegalArgumentException, ClassCastException,
388 NoSuchElementException {
389 this(outExt, new Tuner(config, ExtractorBase.CONFIG_SUFFIX_IE),
390 config.getInt(CONFIG_SENTENCE_TUNE),
391 config.getBoolean(CONFIG_FEEDBACK),
392 config.getBoolean("extract.evaluate"),
393 config.getString("extract.pred.ext"),
394 config.getBoolean("extract.pred.use-outdir"), config);
395 }
396
397 /***
398 * Creates a new instance.
399 *
400 * @param outExt the extension to use for output files
401 * @param theTuner used for TUNE training
402 * @param sentenceTUNE the maximum number of iterations used for TUNE
403 * training the sentence classifier (if used); if 0 or negative, the value
404 * of {@link Tuner#getTuneIterations()} is used
405 * @param giveFeedback if <code>true</code>, a fully incremental setup is
406 * used where the trainer is trained on each document after the extractor
407 * processed it; it's not allowed to set this both this and
408 * {@link Tuner#isTuneEach()} to <code>true</code> when training for
409 * several TUNE iterations because that would mean to evaluate on the
410 * training set
411 * @param doEvaluate whether to evaluate predictions by comparing them to
412 * answer keys, otherwise predictions are stored without evaluating them
413 * @param predExt the extension used to stored predictions (if
414 * <code>doEvaluate</code> is set to <code>false</code>)
415 * @param doStorePredsInOutDir whether to write prediction files to the
416 * configured output directory or the the directory containing the input
417 * file
418 * @param config used to configure superclasses, trainer, and extractor;
419 * if <code>null</code>, the {@linkplain TiesConfiguration#CONF standard
420 * configuration} is used
421 */
422 public TrainEval(final String outExt, final Tuner theTuner,
423 final int sentenceTUNE, final boolean giveFeedback,
424 final boolean doEvaluate, final String predExt,
425 final boolean doStorePredsInOutDir,
426 final TiesConfiguration config) {
427 super(outExt, config);
428
429 tuner = theTuner;
430 sentenceIterations = sentenceTUNE;
431 feedback = giveFeedback;
432 averages = new MultiFMetrics[tuner.getTuneIterations()];
433
434 evaluate = doEvaluate;
435 predExtension = predExt;
436 storePredsInOutDir = doStorePredsInOutDir;
437
438 if (giveFeedback && (tuner.getTuneIterations() > 1)
439 && tuner.isTuneEach()) {
440 throw new IllegalArgumentException(
441 "It's not allowed give feedback "
442 + "when evaluating after each of several iteration because "
443 + "that would mean to evaluate on the training set");
444 }
445
446
447 if (feedback) {
448 feedbackAverages = new ArrayList<MultiFMetrics>();
449 } else {
450 feedbackAverages = null;
451 }
452
453
454
455
456
457 final FieldContainer dummy = new FieldContainer();
458 final MultiFMetrics dummyMetrics = new MultiFMetrics(true);
459 dummyMetrics.storeEntries(dummy);
460 StandardDeviation dummyDev = new StandardDeviation();
461 dummyDev.clear();
462 }
463
464 /***
465 * Check whether to disable sentence training; called after each training
466 * iteration.
467 *
468 * @param trainer the used trainer
469 * @param sentenceMetrics metrics calculated for sentence filtering, if
470 * used (<code>null</code> otherwise)
471 * @param iteration the current iteration
472 */
473 private void checkSentenceTraining(final Trainer trainer,
474 final FMetrics sentenceMetrics, final int iteration) {
475 if ((sentenceIterations > 0) && (sentenceIterations == iteration)) {
476 trainer.disableSentenceTraining();
477 Util.LOG.info("Disabled sentence training after " + iteration
478 + " iterations");
479 }
480
481 if (sentenceMetrics != null) {
482 updateSentenceMetricsStore(sentenceMetrics, true, iteration);
483 }
484 }
485
486 /***
487 * {@inheritDoc}
488 */
489 public void close(final int errorCount)
490 throws IOException, ProcessingException {
491
492 if (errorCount <= 0) {
493 final File outDir = IOUtils.determineOutputDirectory(getConfig());
494
495
496
497
498 final FieldContainer averagesStore =
499 FieldContainer.createFieldContainer(getConfig());
500 boolean storeIterations = (averages.length > 1);
501
502
503 final int lastToStore = Math.min(averages.length,
504 lastUsedTuneIteration);
505
506 for (int i = 1; i <= lastToStore; i++) {
507
508
509
510 if (averages[i - 1] != null && (tuner.isTuneEach()
511 || tuner.getTuneEvaluations().contains(
512 Integer.valueOf(i)) || i == lastToStore)) {
513
514 if (storeIterations) {
515 averagesStore.backgroundMap().put(KEY_ITERATION,
516 Integer.valueOf(i));
517 }
518
519 averages[i - 1].storeEntries(averagesStore);
520 }
521 }
522 averagesStore.storeInFile(outDir, "All", MultiFMetrics.EXT_METRICS,
523 getConfig());
524
525
526 if (feedbackAverages != null) {
527 final FieldContainer feedbackAveragesStore =
528 FieldContainer.createFieldContainer(getConfig());
529
530 for (int i = 0; i < feedbackAverages.size(); i++) {
531 feedbackAverages.get(i).storeEntries(feedbackAveragesStore);
532 }
533 feedbackAveragesStore.storeInFile(outDir, "Feedback",
534 MultiFMetrics.EXT_METRICS, getConfig());
535 }
536 }
537 }
538
539 /***
540 * {@inheritDoc}
541 */
542 protected void doProcess(final Reader reader, final Writer writer,
543 final ContextMap context) throws IOException, ProcessingException {
544
545 runNo++;
546 final String[] files = IOUtils.readURIList(reader);
547
548 if (files.length == 0) {
549 Util.LOG.info("No files to process");
550 } else {
551 final String baseName = IOUtils.getBaseName(
552 new File((String) context.get(KEY_LOCAL_NAME)));
553 final File inDir = (File) context.get(KEY_DIRECTORY);
554 final File outDir = IOUtils.determineOutputDirectory(getConfig());
555 Results currentResults;
556
557
558 MultiFMetricsView currentMetrics = null;
559
560
561 currentResults =
562 trainAndEval(files, inDir, outDir, baseName, writer);
563
564 if (currentResults != null) {
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579 final Iterator<Integer> iterIter = currentResults.iterations();
580 Integer iteration;
581 int it = 0;
582
583 while (iterIter.hasNext()) {
584 iteration = iterIter.next();
585 it = iteration.intValue();
586 currentMetrics =
587 currentResults.getEvaluated(iteration).viewMetrics();
588 if (averages[it-1] == null) {
589 averages[it-1] = new MultiFMetrics(true);
590 }
591 averages[it-1].update(currentMetrics);
592 }
593
594
595
596
597 for (it++; it <= averages.length; it++) {
598 if (averages[it-1] == null) {
599 averages[it-1] = new MultiFMetrics(true);
600 }
601
602
603
604 averages[it-1].update(currentMetrics);
605 }
606 }
607 }
608 }
609
610 /***
611 * Creates and initializes a extractor to use for an evaluation run,
612 * re-using the components of the provided trainer. Subclasses can
613 * overwrite this method to provide a different extractor.
614 *
615 * @param trainer trainer whose components should be re-used
616 * @return the created extractor
617 */
618 protected Extractor initExtractor(final Trainer trainer) {
619
620 return new Extractor(null, trainer);
621 }
622
623 /***
624 * Creates and initializes a trainer to use for an evaluation run,
625 * configured from the
626 * {@link de.fu_berlin.ties.ConfigurableProcessor#getConfig() stored
627 * configuration}. Subclasses can overwrite this method to provide a
628 * different trainer.
629 *
630 * @param runDirectory directory used to run the classifier
631 * @return the created trainer
632 * @throws ProcessingException if an error occurs during initialization
633 */
634 protected Trainer initTrainer(final File runDirectory)
635 throws ProcessingException {
636
637 final Trainer trainer = new Trainer(null, runDirectory, getConfig());
638
639 trainer.reset();
640 return trainer;
641 }
642
643 /***
644 * Returns a string representation of this object.
645 *
646 * @return a textual representation
647 */
648 public String toString() {
649 return new ToStringBuilder(this)
650 .appendSuper(super.toString())
651 .append("tuner", tuner)
652 .append("sentence iterations", sentenceIterations)
653 .append("feedback", feedback)
654 .toString();
655 }
656
657 /***
658 * Helper method for serializing the calculated metrics.
659 *
660 * @param outDirectory directory used to do this run and store the results
661 * @param baseName the base name of the files to use for storing
662 * all extractions and training statistics
663 * @param iteration the number of the current iteration
664 * @param evaluated the container of evaluated extractions to tore
665 * @param sentenceMetrics metrics calculated for sentence filtering, if
666 * used (<code>null</code> otherwise)
667 * @throws IOException if an I/O error occurs while serializing
668 */
669 private void serializeExtractions(final File outDirectory,
670 final String baseName, final int iteration,
671 final EvaluatedExtractionContainer evaluated,
672 final FMetrics sentenceMetrics)
673 throws IOException {
674
675 final FieldContainer resultStorage =
676 FieldContainer.createFieldContainer(getConfig());
677 evaluated.storeEntries(resultStorage);
678 final File evaluatedFile = resultStorage.storeInFile(outDirectory,
679 baseName, Extractor.EXT_EXTRACTIONS, getConfig());
680 Util.LOG.info("Stored results of training + evaluation run in "
681 + evaluatedFile);
682
683
684 if (sentenceMetrics != null) {
685 updateSentenceMetricsStore(sentenceMetrics, false, iteration);
686 }
687 }
688
689 /***
690 * Helper method for serializing the calculated metrics.
691 *
692 * @param outDirectory directory used to do this run and store the results
693 * @param baseName the base name of the files to use for storing
694 * all extractions and training statistics
695 * @param writer used to serialize the calculated metrics
696 * @param results contains the evaluated extractions
697 * @throws IOException if an I/O error occurs while serializing the metrics
698 */
699 private void serializeMetrics(final File outDirectory,
700 final String baseName, final Writer writer,
701 final Results results)
702 throws IOException {
703
704 final FieldContainer metricsStore =
705 FieldContainer.createFieldContainer(getConfig());
706 Integer iteration;
707 EvaluatedExtractionContainer evaluated;
708 final boolean storeIterations = (tuner.getTuneIterations() > 1);
709 Iterator<Integer> tuneIter = results.iterations();
710
711 while (tuneIter.hasNext()) {
712 iteration = tuneIter.next();
713 evaluated = results.getEvaluated(iteration);
714
715
716 if (storeIterations) {
717 metricsStore.backgroundMap().put(KEY_ITERATION, iteration);
718 }
719
720 evaluated.viewMetrics().storeEntries(metricsStore);
721 }
722
723 metricsStore.store(writer);
724 writer.flush();
725
726
727 if (sentenceMetricsStore != null) {
728 sentenceMetricsStore.storeInFile(outDirectory, baseName, "sent",
729 getConfig());
730 }
731 }
732
733 /***
734 * Helper method for serializing the training accuracies.
735 *
736 * @param outDirectory directory used to do this run and store the results
737 * @param baseName the base name of the files to use for storing
738 * all extractions and training statistics
739 * @param accContainers a an of containers used to store the accuracy of
740 * each classifier, might be <code>null</code> if accuracies aren't measured
741 * @param globalAcc an array of accuracies, one for each classifier
742 * @throws IOException if an I/O error occurs while serializing
743 */
744 private void serializeTrainingMetrics(final File outDirectory,
745 final String baseName, final FieldContainer[] accContainers,
746 final AccuracyView[] globalAcc)
747 throws IOException {
748 if (accContainers != null) {
749 String name;
750
751 for (int j = 0; j < globalAcc.length; j++) {
752 if (globalAcc.length > 1) {
753
754 name = baseName + (char) ('a' + j);
755 } else {
756
757 name = baseName;
758 }
759 accContainers[j].storeInFile(outDirectory, name, "train",
760 getConfig());
761 }
762 }
763 }
764
765 /***
766 * Processes an array of files. For each file, a corresponding answer key
767 * (*.ans) must exist.
768 *
769 * @param files the array of file names to process (relative to the
770 * <code>inDirectory</code>)
771 * @param inDirectory directory containing the files to process
772 * @param outDirectory directory used to do this run and store the results
773 * @param baseName the base name of the files to use for storing
774 * all extractions and training statistics
775 * @param writer used to serialize the calculated metrics
776 * @return a wrapper of the results of this run
777 * @throws IOException if an I/O error occurs
778 * @throws ProcessingException if an error occurs during processing
779 */
780 public Results trainAndEval(final String[] files, final File inDirectory,
781 final File outDirectory, final String baseName,
782 final Writer writer) throws IOException, ProcessingException {
783
784 IOUtils.setDefaultDirectory(outDirectory);
785
786
787 final List<String> trainFiles = new LinkedList<String>();
788 final List<String> evalFiles = new LinkedList<String>();
789 tuner.selectFiles(files, trainFiles, evalFiles);
790
791 final long trainStartTime = System.currentTimeMillis();
792 File currentFile;
793 Document currentDoc;
794 ExtractionContainer currentAnswers;
795
796
797 FieldContainer[] accContainers = null;
798 AccuracyView[] currentAcc;
799 AccuracyView[] globalAcc = null;
800 FieldMap accMap;
801 final boolean storeIterations = (tuner.getTuneIterations() > 1);
802
803
804 boolean continueTraining = true;
805 double[] currentOverallAcc;
806 int evalFileNum, feedbackNum;
807
808
809 tuner.reset();
810
811
812 final Trainer trainer = initTrainer(outDirectory);
813 Extractor extractor = null;
814
815
816 FMetrics sentenceMetrics = null;
817 FMetricsView newSentenceMetrics;
818
819 if (trainer.isSentenceFiltering()) {
820 sentenceMetricsStore =
821 FieldContainer.createFieldContainer(getConfig());
822 } else {
823 sentenceMetricsStore = null;
824 }
825
826
827 final Results results = (evaluate) ? new Results() : null;
828
829
830
831
832
833 for (int i = 1; (i <= tuner.getTuneIterations())
834 && continueTraining; i++) {
835 if (tuner.getTuneIterations() > 1) {
836 Util.LOG.debug("Starting TUNE iteration " + i + "/"
837 + tuner.getTuneIterations()
838 + " (will stop if no improvement for "
839 + tuner.getTuneStop() + " iterations)");
840 }
841
842
843
844
845
846
847
848
849 final Iterator trainIter = trainFiles.iterator();
850 trainer.resetGlobalAccuracy();
851
852
853 if (trainer.isSentenceFiltering()) {
854 sentenceMetrics = new FMetrics();
855 }
856
857
858 while (trainIter.hasNext()) {
859 currentFile = IOUtils.resolveFilename(
860 inDirectory, (String) trainIter.next());
861 Util.LOG.debug("Starting to train " + currentFile);
862
863 try {
864 currentDoc =
865 DOMUtils.readDocument(currentFile, getConfig());
866 } catch (DocumentException de) {
867
868 throw new ParsingException("Error while parsing "
869 + currentFile + ": " + de.toString(), de);
870 }
871
872
873 currentAnswers = AnswerBuilder.readCorrespondingAnswerKeys(
874 trainer.getTargetStructure(), currentFile, getConfig());
875
876
877 currentAcc =
878 trainer.train(currentDoc, currentFile, currentAnswers);
879
880
881 if (currentAcc != null) {
882 globalAcc = trainer.viewGlobalAccuracy();
883
884
885 if (accContainers == null) {
886 accContainers = new FieldContainer[globalAcc.length];
887 for (int j = 0; j < globalAcc.length; j++) {
888 accContainers[j] =
889 FieldContainer.createFieldContainer(
890 getConfig());
891 }
892 }
893
894
895 for (int j = 0; j < globalAcc.length; j++) {
896 accMap = globalAcc[j].storeFields();
897 accMap.putAll(currentAcc[j].storeFields());
898 accMap.put(Prediction.KEY_SOURCE,
899 IOUtils.getBaseName(currentFile));
900 if (storeIterations) {
901 accMap.put(KEY_ITERATION, new Integer(i));
902 }
903 accContainers[j].add(accMap);
904 }
905 }
906 Util.LOG.info("Trained " + currentFile);
907
908
909 if (trainer.isSentenceFiltering()) {
910 newSentenceMetrics = trainer.evaluateSentenceFiltering();
911 Util.LOG.debug("Evaluated sentence filtering for trained "
912 + "document: " + newSentenceMetrics);
913 sentenceMetrics.update(newSentenceMetrics);
914 }
915 }
916
917
918 checkSentenceTraining(trainer, sentenceMetrics, i);
919
920
921 if (globalAcc != null) {
922 currentOverallAcc = new double[globalAcc.length];
923
924 for (int j = 0; j < currentOverallAcc.length; j++) {
925 currentOverallAcc[j] = globalAcc[j].getAccuracy();
926 }
927
928 continueTraining = tuner.continueTraining(currentOverallAcc, i);
929 }
930
931 if (tuner.shouldEvaluate(continueTraining, i)) {
932
933 Util.LOG.info("Finished training using " + trainer.toString()
934 + "; " + Util.showDuration(trainStartTime));
935 final long evalStartTime = System.currentTimeMillis();
936 evalFileNum = 0;
937
938
939 if (extractor == null) {
940 extractor = initExtractor(trainer);
941 }
942
943 final Iterator evalIter = evalFiles.iterator();
944 final EvaluatedExtractionContainer evaluated = (evaluate)
945 ? new EvaluatedExtractionContainer(
946 extractor.getTargetStructure(), getConfig())
947 : null;
948 ExtractionContainer currentResults;
949 MultiFMetricsView interimMetrics;
950
951
952 if (extractor.isSentenceFiltering()) {
953 sentenceMetrics = new FMetrics();
954 }
955
956
957 while (evalIter.hasNext()) {
958 currentFile = IOUtils.resolveFilename(inDirectory,
959 (String) evalIter.next());
960 evalFileNum++;
961 Util.LOG.debug("Starting to extract and evaluate file #"
962 + evalFileNum + ": " + currentFile);
963
964 try {
965 currentDoc = DOMUtils.readDocument(currentFile,
966 getConfig());
967 } catch (DocumentException de) {
968
969 throw new ParsingException(de);
970 }
971
972
973 currentResults = extractor.extract(currentDoc, currentFile);
974
975 if (evaluate) {
976
977 currentAnswers =
978 AnswerBuilder.readCorrespondingAnswerKeys(
979 trainer.getTargetStructure(), currentFile,
980 getConfig());
981
982
983 if (feedback) {
984 currentAcc = trainer.train(currentDoc, currentFile,
985 currentAnswers);
986 }
987
988
989
990
991 if (extractor.isSentenceFiltering()) {
992 newSentenceMetrics =
993 extractor.evaluateSentenceFiltering(
994 currentAnswers);
995 Util.LOG.debug("Evaluated sentence filtering for "
996 + "current document: " + newSentenceMetrics);
997 sentenceMetrics.update(newSentenceMetrics);
998 }
999
1000
1001 evaluated.evaluateBatch(currentResults, currentAnswers,
1002 IOUtils.getBaseName(currentFile));
1003
1004
1005 interimMetrics = evaluated.viewMetrics();
1006 Util.LOG.info("Extracted and evaluated " + currentFile
1007 + ", interim results: " + interimMetrics.viewAll());
1008
1009
1010
1011 if (feedback && ((evalFileNum % MEASURE_FEEDBACK == 0)
1012 || !evalIter.hasNext())) {
1013 feedbackNum = new Double(
1014 Math.ceil((double) evalFileNum
1015 / MEASURE_FEEDBACK)).intValue();
1016
1017
1018 if (feedbackAverages.size() < feedbackNum) {
1019 feedbackAverages.add(new MultiFMetrics(true));
1020 }
1021
1022
1023 feedbackAverages.get(feedbackNum - 1).update(
1024 evaluated.viewMetrics());
1025 }
1026 } else {
1027
1028 final File predDir = storePredsInOutDir
1029 ? (outDirectory) : currentFile.getParentFile();
1030 final String localName = IOUtils.getBaseName(
1031 currentFile) + '.' + predExtension;
1032 final File predFile = new File(predDir, localName);
1033 final FieldContainer predStorage =
1034 FieldContainer.createFieldContainer(getConfig());
1035 currentResults.storeEntries(predStorage);
1036 final Writer predWriter =
1037 IOUtils.openWriter(predFile, getConfig());
1038 predStorage.store(predWriter);
1039 predWriter.flush();
1040 IOUtils.tryToClose(predWriter);
1041 Util.LOG.info("Stored predicted extractions in "
1042 + predFile);
1043 }
1044 }
1045
1046 if (evaluate) {
1047 serializeExtractions(outDirectory, baseName, i, evaluated,
1048 sentenceMetrics);
1049 results.addEvaluated(i, evaluated);
1050 }
1051 Util.LOG.info("Finished extraction and evaluation using "
1052 + extractor.toString() + "; "
1053 + Util.showDuration(evalStartTime));
1054 }
1055
1056
1057 lastUsedTuneIteration = Math.max(lastUsedTuneIteration, i);
1058 }
1059
1060
1061 serializeTrainingMetrics(outDirectory, baseName, accContainers,
1062 globalAcc);
1063
1064 if (evaluate) {
1065 serializeMetrics(outDirectory, baseName, writer, results);
1066
1067
1068 }
1069
1070 extractor.destroy();
1071 return results;
1072 }
1073
1074 /***
1075 * Adds the results of a training or evaluation iteration to the statistics
1076 * calculated for sentence filtering.
1077 *
1078 * @param newMetrics the metrics to add
1079 * @param isTraining whether this is a training or an evaluation result
1080 * @param iteration the number of the iteration (counting from 1)
1081 */
1082 private void updateSentenceMetricsStore(final FMetricsView newMetrics,
1083 final boolean isTraining, final int iteration) {
1084 final FieldMap fields = newMetrics.storeFields();
1085
1086
1087 fields.put(KEY_TYPE, isTraining ? TYPE_TRAIN : TYPE_EVAL);
1088 fields.put(KEY_ITERATION, new Integer(iteration));
1089 sentenceMetricsStore.add(fields);
1090 }
1091
1092 }