diff --git a/.gitignore b/.gitignore index 2531f82e..6be03663 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,7 @@ *.iml target/ /gui/hs_err_pid*.log +**/.vscode +**/.settings +**/.classpath +**/.project \ No newline at end of file diff --git a/README.md b/README.md index 67c1a65e..8b2fc31f 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,8 @@ by Paul Vorbach (German). [![Donate with PayPal](https://www.paypalobjects.com/en_US/i/btn/btn_donate_SM.gif)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=LF8T2JP2YUUUE) +## Usage +Basic usage is documented on [our wiki page](https://github.com/tesseract4java/tesseract4java/wiki/Usage) ## Download @@ -55,23 +57,19 @@ Batch export functionality to handle large projects This software is written in Java and can be built using [Apache Maven]. In order to build the software you have to follow these steps: - 1. Obtain a copy either by cloning the repository or downloading the current [zip file][zip]. - 2. Also obtain a copy of a [patched version of ocrevalUAtion][ocrevalUAtion patched] ([zip file][ocrevalUAtion zip]). - 3. Open a command line in the ocrevalUAtion directory and run `mvn clean install`. - 4. `cd` to the tesseract4java directory and run `mvn clean package -Pstandalone`. This will include the - Tesseract binaries for your platform. You can manually define the platform by providing the option - `-Djavacpp.platform=[PLATFORM]` (available platforms are `windows-x86_64`, `windows-x86`, `linux-x86_64`, `linux-x86`, - and `macosx-x86_64`). + 1. `git clone https://github.com/tesseract4java/tesseract4java.git` + 2. `cd tesseract4java` + 3. `git submodule init` + 4. `git submodule update` + 5. `mvn clean package -Pstandalone`. This will include the Tesseract binaries for your platform. You can manually + define the platform by providing the option `-Djavacpp.platform=[PLATFORM]` (available platforms are + `windows-x86_64`, `windows-x86`, `linux-x86_64`, `linux-x86`, and `macosx-x86_64`). -After you've run through all steps, the directory "tesseract4java/gui/target" will contain the file +After you've run through all steps, the directory "gui/target" will contain the file "tesseract4java-[VERSION]-[PLATFORM].jar", which you can run by double-clicking or executing `java -jar tesseract4java-[VERSION]-[PLATFORM].jar`. [Apache Maven]: https://maven.apache.org/ -[zip]: https://github.com/tesseract4java/tesseract4java/archive/develop.zip -[ocrevalUAtion patched]: https://github.com/tesseract4java/ocrevalUAtion -[ocrevalUAtion zip]: https://github.com/tesseract4java/ocrevalUAtion/archive/master.zip - ## Credits @@ -95,7 +93,7 @@ GPLv3 ~~~ tesseract4java - a graphical user interface for the Tesseract OCR engine -Copyright (C) 2014-2016 Paul Vorbach +Copyright (C) 2014-2019 Paul Vorbach This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/gui/pom.xml b/gui/pom.xml index fa41ba33..d3f526fc 100644 --- a/gui/pom.xml +++ b/gui/pom.xml @@ -5,9 +5,9 @@ 4.0.0 - de.vorb + de.vorb.tesseract tesseract4java - 0.2.0-SNAPSHOT + 0.3.0-SNAPSHOT gui @@ -23,19 +23,18 @@ 19.0 - de.vorb + de.vorb.tesseract tools - ${project.version} com.twelvemonkeys.imageio imageio-tiff - 3.1.1 + 3.4.1 junit junit - 4.12 + 4.13.1 test diff --git a/gui/src/main/java/de/vorb/tesseract/gui/controller/TesseractController.java b/gui/src/main/java/de/vorb/tesseract/gui/controller/TesseractController.java index d11f7db7..dc6a3513 100644 --- a/gui/src/main/java/de/vorb/tesseract/gui/controller/TesseractController.java +++ b/gui/src/main/java/de/vorb/tesseract/gui/controller/TesseractController.java @@ -1,5 +1,6 @@ package de.vorb.tesseract.gui.controller; +import com.google.common.collect.Lists; import de.vorb.tesseract.gui.io.BoxFileReader; import de.vorb.tesseract.gui.io.BoxFileWriter; import de.vorb.tesseract.gui.io.PlainTextWriter; @@ -49,8 +50,6 @@ import de.vorb.tesseract.util.TraineddataFiles; import de.vorb.tesseract.util.feat.Feature3D; import de.vorb.util.FileNames; - -import com.google.common.collect.Lists; import eu.digitisation.input.Batch; import eu.digitisation.input.Parameters; import eu.digitisation.input.WarningException; @@ -231,6 +230,8 @@ public TesseractController() { try { pageRecognitionProducer.init(); + pageRecognitionProducer.setPageSegmentationMode(getPageSegmentationMode()); + } catch (IOException e) { e.printStackTrace(); } @@ -1141,6 +1142,21 @@ private void handlePreferences() { final String editorFont = (String) prefDialog.getComboEditorFont().getSelectedItem(); globalPrefs.put(PreferencesDialog.KEY_EDITOR_FONT, editorFont); + // Update the page segmentation mode if necessary + int currentPageSegMode = getPageSegmentationMode(); + int pageSegMode = prefDialog.getPageSegmentationMode(); + boolean hasPageSegModeChanged = currentPageSegMode != pageSegMode; + if (hasPageSegModeChanged) { + globalPrefs.putInt(PreferencesDialog.KEY_PAGE_SEG_MODE, pageSegMode); + pageRecognitionProducer.setPageSegmentationMode(pageSegMode); + + // Update model with new segmentation mode + if (activeComponent instanceof PageModelComponent) { + final Optional pm = ((PageModelComponent) activeComponent).getPageModel(); + pm.ifPresent(it -> setImageModel(Optional.of(it.getImageModel()))); + } + } + view.getRecognitionPane().setRenderingFont(renderingFont); if (view.getActiveComponent() == view.getRecognitionPane()) { view.getRecognitionPane().render(); @@ -1241,6 +1257,10 @@ private void handleTesseractTrainer() { trainer.setVisible(true); } + private int getPageSegmentationMode() { + return PreferencesUtil.getPreferences().getInt(PreferencesDialog.KEY_PAGE_SEG_MODE, PreferencesDialog.DEFAULT_PSM_MODE); + } + public void setPageModel(Optional model) { if (projectModel.isPresent() && model.isPresent()) { try { diff --git a/gui/src/main/java/de/vorb/tesseract/gui/view/dialogs/PreferencesDialog.java b/gui/src/main/java/de/vorb/tesseract/gui/view/dialogs/PreferencesDialog.java index ba9e8e61..f54c69ab 100644 --- a/gui/src/main/java/de/vorb/tesseract/gui/view/dialogs/PreferencesDialog.java +++ b/gui/src/main/java/de/vorb/tesseract/gui/view/dialogs/PreferencesDialog.java @@ -26,15 +26,35 @@ public class PreferencesDialog extends JDialog { private static final long serialVersionUID = 1L; + private static final String[] PSM_MODES = { + "0 - PSM_OSD_ONLY", + "1 - PSM_AUTO_OSD", + "2 - PSM_AUTO_ONLY", + "3 - (DEFAULT) PSM_AUTO", + "4 - PSM_SINGLE_COLUMN", + "5 - PSM_SINGLE_BLOCK_VERT_TEXT", + "6 - PSM_SINGLE_BLOCK", + "7 - PSM_SINGLE_LINE", + "8 - PSM_SINGLE_WORD", + "9 - PSM_CIRCLE_WORD", + "10 - PSM_SINGLE_CHAR", + "11 - PSM_SPARSE_TEXT", + "12 - PSM_SPARSE_TEXT_OSD", + "13 - PSM_RAW_LINE", + }; + public static final int DEFAULT_PSM_MODE = 3; + public static final String KEY_LANGDATA_DIR = "langdata_dir"; public static final String KEY_RENDERING_FONT = "rendering_font"; public static final String KEY_EDITOR_FONT = "editor_font"; + public static final String KEY_PAGE_SEG_MODE = "page_seg_mode"; private final JPanel contentPanel = new JPanel(); private JTextField tfLangdataDir; private final JComboBox comboRenderingFont; private final JComboBox comboEditorFont; + private final JComboBox comboPageSegMode; private ResultState resultState = ResultState.CANCEL; @@ -134,48 +154,20 @@ public PreferencesDialog() { final GraphicsEnvironment graphicsEnvironment = GraphicsEnvironment.getLocalGraphicsEnvironment(); final String[] availableFontFamilyNames = graphicsEnvironment.getAvailableFontFamilyNames(); - { - JLabel lblRenderingFont = new JLabel("Rendering font:"); - GridBagConstraints gbc_lblRenderingFont = new GridBagConstraints(); - gbc_lblRenderingFont.anchor = GridBagConstraints.EAST; - gbc_lblRenderingFont.insets = new Insets(0, 0, 0, 5); - gbc_lblRenderingFont.gridx = 0; - gbc_lblRenderingFont.gridy = 2; - contentPanel.add(lblRenderingFont, gbc_lblRenderingFont); - } - { - comboRenderingFont = new JComboBox<>(); - GridBagConstraints gbc_comboRenderingFont = new GridBagConstraints(); - gbc_comboRenderingFont.insets = new Insets(0, 0, 0, 5); - gbc_comboRenderingFont.fill = GridBagConstraints.HORIZONTAL; - gbc_comboRenderingFont.gridx = 1; - gbc_comboRenderingFont.gridy = 2; - contentPanel.add(comboRenderingFont, gbc_comboRenderingFont); - - Arrays.stream(availableFontFamilyNames).forEach(comboRenderingFont::addItem); - comboRenderingFont.setSelectedItem(pref.get(PreferencesDialog.KEY_RENDERING_FONT, Font.SANS_SERIF)); - } - { - JLabel lblEditorFont = new JLabel("Editor font:"); - GridBagConstraints gbc_lblEditorFont = new GridBagConstraints(); - gbc_lblEditorFont.anchor = GridBagConstraints.EAST; - gbc_lblEditorFont.insets = new Insets(0, 0, 0, 5); - gbc_lblEditorFont.gridx = 0; - gbc_lblEditorFont.gridy = 3; - contentPanel.add(lblEditorFont, gbc_lblEditorFont); - } - { - comboEditorFont = new JComboBox<>(); - GridBagConstraints gbc_comboEditorFont = new GridBagConstraints(); - gbc_comboEditorFont.insets = new Insets(0, 0, 0, 5); - gbc_comboEditorFont.fill = GridBagConstraints.HORIZONTAL; - gbc_comboEditorFont.gridx = 1; - gbc_comboEditorFont.gridy = 3; - contentPanel.add(comboEditorFont, gbc_comboEditorFont); - - Arrays.stream(availableFontFamilyNames).forEach(comboEditorFont::addItem); - comboEditorFont.setSelectedItem(pref.get(PreferencesDialog.KEY_EDITOR_FONT, Font.MONOSPACED)); - } + // Rendering font + addGridLabel(0, 2, "Rendering font:"); + final String initialFontFamilyName = pref.get(PreferencesDialog.KEY_EDITOR_FONT, Font.SANS_SERIF); + comboRenderingFont = createGridComboBox(1, 2, availableFontFamilyNames, initialFontFamilyName); + + // Editor font + addGridLabel(0, 3, "Editor font:"); + final String initialEditorFontFamilyName = pref.get(PreferencesDialog.KEY_RENDERING_FONT, Font.MONOSPACED); + comboEditorFont = createGridComboBox(1, 3, availableFontFamilyNames, initialEditorFontFamilyName); + + // Page Segmentation Modes + addGridLabel(0, 4, "Page Segmentation Mode:"); + final int pageSegMode = pref.getInt(PreferencesDialog.KEY_PAGE_SEG_MODE, DEFAULT_PSM_MODE); + comboPageSegMode = createGridComboBox(1, 4, PSM_MODES, PSM_MODES[pageSegMode]); pack(); setMinimumSize(getSize()); @@ -197,9 +189,39 @@ public JComboBox getComboEditorFont() { return comboEditorFont; } + public int getPageSegmentationMode() { + return comboPageSegMode.getSelectedIndex(); + } + public ResultState showPreferencesDialog(Component parent) { setLocationRelativeTo(parent); setVisible(true); return resultState; } + + private Component addGridLabel(int gridX, int gridY, String label) { + JLabel jLabel = new JLabel(label); + GridBagConstraints constraints = new GridBagConstraints(); + constraints.anchor = GridBagConstraints.EAST; + constraints.insets = new Insets(0, 0, 0, 5); + constraints.gridx = gridX; + constraints.gridy = gridY; + contentPanel.add(jLabel, constraints); + return jLabel; + } + + private JComboBox createGridComboBox(int gridX, int gridY, T[] options, T selectedItem) { + JComboBox jComboBox = new JComboBox<>(); + GridBagConstraints constraints = new GridBagConstraints(); + constraints.insets = new Insets(0, 0, 0, 5); + constraints.fill = GridBagConstraints.HORIZONTAL; + constraints.gridx = gridX; + constraints.gridy = gridY; + contentPanel.add(jComboBox, constraints); + + Arrays.stream(options).forEach(jComboBox::addItem); + jComboBox.setSelectedItem(selectedItem); + return jComboBox; + } + } diff --git a/gui/src/main/java/de/vorb/tesseract/gui/work/PageRecognitionProducer.java b/gui/src/main/java/de/vorb/tesseract/gui/work/PageRecognitionProducer.java index 11acc9a1..bb3e50ed 100644 --- a/gui/src/main/java/de/vorb/tesseract/gui/work/PageRecognitionProducer.java +++ b/gui/src/main/java/de/vorb/tesseract/gui/work/PageRecognitionProducer.java @@ -30,6 +30,7 @@ public class PageRecognitionProducer extends RecognitionProducer { private final TesseractController controller; private final HashMap variables = new HashMap<>(); + private int pageSegmentationMode = tesseract.PSM_AUTO; public PageRecognitionProducer(TesseractController controller, Path tessdataDir, String trainingFile) { @@ -67,7 +68,7 @@ public void reset() throws IOException { tesseract.OEM_DEFAULT); // set page segmentation mode - tesseract.TessBaseAPISetPageSegMode(getHandle(), tesseract.PSM_AUTO); + tesseract.TessBaseAPISetPageSegMode(getHandle(), pageSegmentationMode); // set variables for (Entry var : variables.entrySet()) { @@ -80,6 +81,11 @@ public void close() throws IOException { tesseract.TessBaseAPIDelete(getHandle()); } + public void setPageSegmentationMode(int pageSegmentationMode) { + this.pageSegmentationMode = pageSegmentationMode; + tesseract.TessBaseAPISetPageSegMode(getHandle(), pageSegmentationMode); + } + public void loadImage(Path imageFile) { if (lastPix.isPresent()) { // destroy old pix diff --git a/ocrevalUAtion/AUTHORS b/ocrevalUAtion/AUTHORS new file mode 100644 index 00000000..adacc4a7 --- /dev/null +++ b/ocrevalUAtion/AUTHORS @@ -0,0 +1 @@ +Rafael C. Carrasco (carrasco@ua.es) \ No newline at end of file diff --git a/ocrevalUAtion/README.md b/ocrevalUAtion/README.md new file mode 100644 index 00000000..bcf32a8a --- /dev/null +++ b/ocrevalUAtion/README.md @@ -0,0 +1,25 @@ +ocrevalUAtion [![Build Status](https://secure.travis-ci.org/impactcentre/ocrevalUAtion.png?branch=master)](http://travis-ci.org/impactcentre/ocrevalUAtion) +============= + +This set of classes provides basic support to perform the comparison of +two text files: a reference file (a ground-truth document) and a the output from an OCR engine (a text file). + +Options for specific behavior include: ignore case, ignore diacritics, +ignore punctuation, ignore stop-words, Unicode and user-defined equivalences between characters. + +It can be used with the graphic user interface (GUI) provided, in addition to command line interface usage. + +Supported input formats include: plain text, FineReader 10 XML, PAGE XML, ALTO XML and hOCR HTML. + +The output generates a report with statistics (including CER and WER error rates) +and a table with the parallell input texts where the differences are highlighted. + +A gentle introduction to OCR evaluation and to this tool can be found at https://sites.google.com/site/textdigitisation/ + +You can download the latest release from [here](https://bintray.com/impactocr/maven/ocrevalUAtion). + +Instructions on how to use ocrevalUAtion can be found in the [wiki](https://github.com/impactcentre/ocrevalUAtion/wiki). + + + + diff --git a/ocrevalUAtion/pom.xml b/ocrevalUAtion/pom.xml new file mode 100644 index 00000000..45b5a58e --- /dev/null +++ b/ocrevalUAtion/pom.xml @@ -0,0 +1,270 @@ + + 4.0.0 + + de.vorb.tesseract + tesseract4java + 0.3.0-SNAPSHOT + + ocrevalUAtion + ocrevalUAtion + jar + OCR Evaluation Tool + + IMPACT Centre of Competence + http://www.digitisation.eu/ + + 2009 + + scm:git:https://github.com/impactcentre/ocrevalUAtion.git + scm:git:git@github.com:impactcentre/ocrevalUAtion.git + https://github.com/impactcentre/ocrevalUAtion + HEAD + + + + GNU General Public License 2.0 + http://www.gnu.org/licenses/gpl-2.0.html + + + + UTF-8 + 1.7 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + 1.7 + 1.7 + true + + + + org.apache.maven.plugins + maven-javadoc-plugin + 2.9.1 + + ${basedir}/api + api + + + + org.apache.maven.plugins + maven-jar-plugin + 2.4 + + + + true + eu.digitisation.input.GUI + + + + + + maven-assembly-plugin + 2.4 + + ocrevaluation + + jar-with-dependencies + + false + + + true + eu.digitisation.input.GUI + + + + + + make-assembly + package + + single + + + + + + maven-resources-plugin + 2.6 + + + copy-resources + validate + + copy-resources + + + ${basedir}/target + + + . + true + + userProperties.xml + + + + + + + + + + + maven-deploy-plugin + 2.8.1 + + internal.repo::default::file://${project.build.directory}/mvn-repo + + + + + + + + internal.repo + Temporary Staging Repository + file://${project.build.directory}/mvn-repo + + + + + junit + junit + 4.13.1 + test + + + + javax.media.jai + com.springsource.javax.media.jai.core + 1.1.3 + + + javax.media.jai + com.springsource.javax.media.jai.codec + 1.1.3 + + + org.jsoup + jsoup + 1.14.2 + + + org.apache.tika + tika-parsers + 1.4 + jar + + + org.apache.geronimo.specs + geronimo-stax-api_1.0_spec + + + org.gagravarr + vorbis-java-core + + + xml-apis + xml-apis + + + rome + rome + + + + + jakarta.xml.bind + jakarta.xml.bind-api + 2.3.3 + + + + + + com.springsource.repository.bundles.external + SpringSource Enterprise Bundle Repository - External Bundle Releases + https://repository.springsource.com/maven/bundles/external + + + diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/File2Text.java b/ocrevalUAtion/src/main/java/eu/digitisation/File2Text.java new file mode 100644 index 00000000..e55a5d70 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/File2Text.java @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation; + +import eu.digitisation.input.SchemaLocationException; +import eu.digitisation.input.WarningException; +import eu.digitisation.text.CharFilter; +import eu.digitisation.text.Text; +import java.io.File; + +/** + * + * @author R.C.C + */ +public class File2Text { + + public static void main(String[] args) throws WarningException, + SchemaLocationException { + if (args.length > 0) { + File file = new File(args[0]); + CharFilter filter = null; + if (args.length > 1) { + filter = new CharFilter(new File(args[1])); + } + Text content = new Text(file); + System.out.println(content.toString(filter)); + } + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/Main.java b/ocrevalUAtion/src/main/java/eu/digitisation/Main.java new file mode 100644 index 00000000..343b7312 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/Main.java @@ -0,0 +1,90 @@ +package eu.digitisation; + +import eu.digitisation.input.Batch; +import eu.digitisation.input.Parameters; +import eu.digitisation.input.SchemaLocationException; +import eu.digitisation.input.WarningException; +import eu.digitisation.log.Messages; +import eu.digitisation.output.Report; +import java.io.File; +import java.io.InvalidObjectException; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Main class for ocrevalUAtion: version 0.92 + */ +public class Main { + + static final String helpMsg = "Usage:\t" + + "ocrevalUAtion -gt file1" + + "-ocr file2" + + "[-e encoding]" + + " [-o output_file_or_dir ] [-r equivalences_file]" + + " [-c] [-ic] [-id] [-ip]"; + + private static void exit_gracefully() { + System.err.println(helpMsg); + System.exit(0); + } + + /** + * @param args the command line arguments + * @throws eu.digitisation.input.WarningException + */ + public static void main(String[] args) throws WarningException { + Parameters pars = new Parameters(); + File workingDirectory; + + // Read parameters (String switch needs Java 1.7 or later) + for (int n = 0; n < args.length; ++n) { + String arg = args[n]; + if (arg.equals("-h")) { + exit_gracefully(); + } else if (arg.equals("-gt")) { + pars.gtfile.setValue(new File(args[++n])); + } else if (arg.equals("-e")) { + pars.encoding.setValue(args[++n]); + } else if (arg.equals("-ocr")) { + pars.ocrfile.setValue(new File(args[++n])); + } else if (arg.equals("-r")) { + pars.eqfile.setValue(new File(args[++n])); + } else if (arg.equals("-o")) { + pars.outfile.setValue(new File(args[++n])); + } else if (arg.equals("-c")) { + pars.compatibility.setValue(true); + } else if (arg.equals("-ic")) { + pars.ignoreCase.setValue(true); + } else if (arg.equals("-id")) { + pars.ignoreDiacritics.setValue(true); + } else if (arg.equals("-ip")) { + pars.ignorePunctuation.setValue(true); + } else { + System.err.println("Unrecognized option " + arg); + exit_gracefully(); + } + } + + if (pars.gtfile.getValue() == null || pars.ocrfile.getValue() == null) { + System.err.println("Not enough arguments"); + exit_gracefully(); + } + + if (pars.outfile.getValue() == null) { + String name = pars.ocrfile.getValue().getName().replaceAll("\\.\\w+", "") + + "_report.html"; + pars.outfile.setValue(new File(pars.ocrfile.getValue().getParent(), name)); + } + + try { + Batch batch = new Batch(pars.gtfile.getValue(), pars.ocrfile.getValue()); + Report report = new Report(batch, pars); + report.write(pars.outfile.getValue()); + } catch (InvalidObjectException ex) { + Messages.info(Main.class.getName() + ": " + ex); + } catch (SchemaLocationException ex) { + Messages.info(Main.class.getName() + ": " + ex); + } + + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/distance/Aligner.java b/ocrevalUAtion/src/main/java/eu/digitisation/distance/Aligner.java new file mode 100644 index 00000000..125ee5cc --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/distance/Aligner.java @@ -0,0 +1,460 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.distance; + +import eu.digitisation.text.CharMap; +import eu.digitisation.text.Text; +import eu.digitisation.xml.DocumentBuilder; +import java.io.File; +import org.w3c.dom.Element; + +/** + * Alignments between 2 texts (output in XHTML format) + * + * @author R.C.C + */ +public class Aligner { + + // style for unaligned segments + final static String uStyle = "background-color:aquamarine"; + // style for highlight replacement in parallel text + final static String twinStyle = ""; + + /** + * @return 3-wise minimum. + */ + private static int min(int x, int y, int z) { + return Math.min(x, Math.min(y, z)); + } + + /** + * Compute the table of basic edit operations needed to transform first into + * second + * + * @param first + * source string + * @param second + * target string + * @return the table of minimal basic edit operations needed to transform + * first into second + */ + private static EditTable alignTab(String first, String second) { + int l1; // length of first + int l2; // length of second + int[][] A; // distance table + EditTable B; // edit operations + + // intialize + l1 = first.length(); + l2 = second.length(); + A = new int[2][second.length() + 1]; + B = new EditTable(first.length() + 1, second.length() + 1); + // Compute first row + A[0][0] = 0; + B.set(0, 0, EdOp.KEEP); + for (int j = 1; j <= second.length(); ++j) { + A[0][j] = A[0][j - 1] + 1; + B.set(0, j, EdOp.INSERT); + } + + // Compute other rows + for (int i = 1; i <= first.length(); ++i) { + char c1 = first.charAt(l1 - i); + A[i % 2][0] = A[(i - 1) % 2][0] + 1; + B.set(i, 0, EdOp.DELETE); + for (int j = 1; j <= second.length(); ++j) { + char c2 = second.charAt(l2 - j); + + if (c1 == c2) { + A[i % 2][j] = A[(i - 1) % 2][j - 1]; + B.set(i, j, EdOp.KEEP); + } else if (Character.isSpaceChar(c1) + ^ Character.isSpaceChar(c2)) { + // No alignment between blank and not-blank + if (A[(i - 1) % 2][j] < A[i % 2][j - 1]) { + A[i % 2][j] = A[(i - 1) % 2][j] + 1; + B.set(i, j, EdOp.DELETE); + } else { + A[i % 2][j] = A[i % 2][j - 1] + 1; + B.set(i, j, EdOp.INSERT); + } + } else { + A[i % 2][j] = min(A[(i - 1) % 2][j] + 1, + A[i % 2][j - 1] + 1, + A[(i - 1) % 2][j - 1] + 1); + if (A[i % 2][j] == A[(i - 1) % 2][j] + 1) { + B.set(i, j, EdOp.DELETE); + } else if (A[i % 2][j] == A[i % 2][j - 1] + 1) { + B.set(i, j, EdOp.INSERT); + } else { + B.set(i, j, EdOp.SUBSTITUTE); + } + } + } + } + return B; + } + + /** + * A minimal sequence of edit operations transforming the first string into + * the second + * + * @param first + * the first string + * @param second + * the second string + * @return a minimal sequence of edit operations transforming the first + * string into the second + */ + public static EditSequence path(String first, String second) { + return alignTab(first, second).path(); + } + + /** + * Shows text alignment based on a pseudo-Levenshtein distance where + * white-spaces are not allowed to be replaced with text or vice-versa + * + * @param header1 + * first text title for table head + * @param header2 + * second text title for table head + * @param first + * the first text + * @param second + * the second text + * @param map + * a CharMap for character equivalences + * @return a table in XHTML format showing the alignments + */ + public static Element alignmentMap(String header1, String header2, + String first, String second, CharMap map) { + EditTable B = (map == null) + ? alignTab(first, second) + : alignTab(map.normalForm(first), map.normalForm(second)); + DocumentBuilder builder = new DocumentBuilder("table"); + Element table = builder.root(); + Element row; + Element cell1; + Element cell2; + int l1; + int l2; + int len; + int i; + int j; + String s1; + String s2; + + // features + table.setAttribute("border", "1"); + // content + row = builder.addElement("tr"); + cell1 = builder.addElement(row, "td"); + cell2 = builder.addElement(row, "td"); + cell1.setAttribute("width", "50%"); + cell2.setAttribute("width", "50%"); + cell1.setAttribute("align", "center"); + cell2.setAttribute("align", "center"); + builder.addTextElement(cell1, "h3", header1); + builder.addTextElement(cell2, "h3", header2); + row = builder.addElement("tr"); + cell1 = builder.addElement(row, "td"); + cell2 = builder.addElement(row, "td"); + + l1 = first.length(); + l2 = second.length(); + i = l1; + j = l2; + while (i > 0 && j > 0) { + switch (B.get(i, j)) { + case KEEP: + len = 1; + while (len < i && len < j + && B.get(i - len, j - len) == EdOp.KEEP) { + ++len; + } + s1 = first.substring(l1 - i, l1 - i + len); + s2 = second.substring(l2 - j, l2 - j + len); + builder.addText(cell1, s1); + builder.addText(cell2, s2); + i -= len; + j -= len; + break; + case DELETE: + len = 1; + while (len < i && B.get(i - len, j) == EdOp.DELETE) { + ++len; + } + s1 = first.substring(l1 - i, l1 - i + len); + builder.addTextElement(cell1, "font", s1) + .setAttribute("style", uStyle); + i -= len; + break; + case INSERT: + len = 1; + + while (len < j && B.get(i, j - len) == EdOp.INSERT) { + ++len; + } + s2 = second.substring(l2 - j, l2 - j + len); + builder.addTextElement(cell2, "font", s2) + .setAttribute("style", uStyle); + j -= len; + break; + case SUBSTITUTE: + len = 1; + while (len < i && len < j + && B.get(i - len, j - len) == EdOp.SUBSTITUTE) { + ++len; + } + s1 = first.substring(l1 - i, l1 - i + len); + s2 = second.substring(l2 - j, l2 - j + len); + Element span1 = builder.addElement(cell1, "span"); + Element span2 = builder.addElement(cell2, "span"); + String id1 = "l" + i + "." + j; + String id2 = "r" + i + "." + j; + span1.setAttribute("title", s2); + span2.setAttribute("title", s1); + span1.setAttribute("id", id1); + span2.setAttribute("id", id2); + span1.setAttribute("onmouseover", + "document.getElementById('" + + id2 + "').style.background='greenyellow'"); + span2.setAttribute("onmouseover", + "document.getElementById('" + + id1 + "').style.background='greenyellow'"); + span1.setAttribute("onmouseout", + "document.getElementById('" + + id2 + "').style.background='none'"); + span2.setAttribute("onmouseout", + "document.getElementById('" + + id1 + "').style.background='none'"); + builder.addTextElement(span1, "font", s1) + .setAttribute("color", "red"); + builder.addTextElement(span2, "font", s2) + .setAttribute("color", "red"); + i -= len; + j -= len; + break; + } + } + if (i > 0) { + s1 = first.substring(l1 - i, l1); + builder.addTextElement(cell1, "font", s1) + .setAttribute("style", uStyle); + + } + if (j > 0) { + s2 = second.substring(l2 - j, l2); + builder.addTextElement(cell2, "font", s2) + .setAttribute("style", uStyle); + } + return builder.document().getDocumentElement(); + } + + /** + * Shows text alignment based on a pseudo-Levenshtein distance where + * white-spaces are not allowed to be replaced with text or vice-versa + * + * @param header1 + * first text title for table head + * @param header2 + * second text title for table head + * @param first + * the first text + * @param second + * the second text + * @return a table in XHTML format showing the alignments + */ + public static Element alignmentMap(String header1, String header2, + String first, String second) { + return alignmentMap(header1, header2, first, second, null); + } + + /** + * Shows text alignment based on a pseudo-Levenshtein distance where + * white-spaces are not allowed to be replaced with text or vice-versa + * + * @param header1 + * first text title for table head + * @param header2 + * second text title for table head + * @param first + * the first text + * @param second + * the second text + * @param w + * the weighs associated to basic edit operations + * @return a table in XHTML format showing the alignments + */ + public static Element bitext(String header1, String header2, + String first, String second, EdOpWeight w, EditSequence edition) { + DocumentBuilder builder = new DocumentBuilder("table"); + Element table = builder.root(); + Element row; + Element cell1; + Element cell2; + int l1; + int l2; + int len; + int i; + int j; + String s1; + String s2; + + // features + table.setAttribute("border", "1"); + // content + row = builder.addElement("tr"); + cell1 = builder.addElement(row, "td"); + cell2 = builder.addElement(row, "td"); + cell1.setAttribute("width", "50%"); + cell2.setAttribute("width", "50%"); + cell1.setAttribute("align", "center"); + cell2.setAttribute("align", "center"); + builder.addTextElement(cell1, "h3", header1 + " (Transcription)"); + builder.addTextElement(cell2, "h3", header2 + " (OCR Result)"); + row = builder.addElement("tr"); + cell1 = builder.addElement(row, "td"); + cell2 = builder.addElement(row, "td"); + + l1 = first.length(); + l2 = second.length(); + i = 0; + j = 0; + len = 0; + + for (int n = 0; n < edition.size(); n += len) { + EdOp op = edition.get(n); + + // free rides first + if (op == EdOp.DELETE && w.del(first.charAt(i)) == 0) { + builder.addText(cell1, first.substring(i, i + 1)); + ++i; + len = 1; + } else if (op == EdOp.INSERT && w.ins(second.charAt(j)) == 0) { + builder.addText(cell2, second.substring(j, j + 1)); + ++j; + len = 1; + } else if (op == EdOp.SUBSTITUTE + && w.sub(first.charAt(i), second.charAt(j)) == 0) { + builder.addText(cell1, first.substring(i, i + 1)); + builder.addText(cell2, second.substring(j, j + 1)); + ++i; + ++j; + len = 1; + } else { + switch (op) { + case KEEP: + len = 1; + while (i + len < l1 && j + len < l2 + && edition.get(n + len) == EdOp.KEEP) { + ++len; + } + s1 = first.substring(i, i + len); + s2 = second.substring(j, j + len); + builder.addText(cell1, s1); + builder.addText(cell2, s2); + i += len; + j += len; + break; + case DELETE: + len = 1; + while (i + len < l1 + && edition.get(n + len) == EdOp.DELETE + && w.del(first.charAt(i + len)) > 0) { + ++len; + } + s1 = first.substring(i, i + len); + builder.addTextElement(cell1, "font", s1) + .setAttribute("style", uStyle); + i += len; + break; + + case INSERT: + len = 1; + while (j + len < l2 + && edition.get(n + len) == EdOp.INSERT + && w.ins(second.charAt(j + len)) > 0) { + ++len; + } + s2 = second.substring(j, j + len); + builder.addTextElement(cell2, "font", s2) + .setAttribute("style", uStyle); + j += len; + break; + case SUBSTITUTE: + len = 1; + while (i + len < l1 + && j + len < l2 + && edition.get(n + len) == EdOp.SUBSTITUTE + && w.sub(first.charAt(i + len), + second.charAt(j + len)) > 0) { + ++len; + } + s1 = first.substring(i, i + len); + s2 = second.substring(j, j + len); + Element span1 = builder.addElement(cell1, "span"); + Element span2 = builder.addElement(cell2, "span"); + String id1 = "l" + i + "." + j; + String id2 = "r" + i + "." + j; + span1.setAttribute("title", s2); + span2.setAttribute("title", s1); + span1.setAttribute("id", id1); + span2.setAttribute("id", id2); + span1.setAttribute("onmouseover", + "document.getElementById('" + + id2 + "').style.background='greenyellow'"); + span2.setAttribute("onmouseover", + "document.getElementById('" + + id1 + "').style.background='greenyellow'"); + span1.setAttribute("onmouseout", + "document.getElementById('" + + id2 + "').style.background='none'"); + span2.setAttribute("onmouseout", + "document.getElementById('" + + id1 + "').style.background='none'"); + builder.addTextElement(span1, "font", s1) + .setAttribute("color", "red"); + builder.addTextElement(span2, "font", s2) + .setAttribute("color", "red"); + i += len; + j += len; + break; + } + } + } + return builder.document().getDocumentElement(); + } + + public static void main(String[] args) + throws Exception { + File f1 = new File(args[0]); + File f2 = new File(args[1]); + File ofile = new File("/tmp/out.html"); + + String s1 = new Text(f1).toString(); + String s2 = new Text(f2).toString(); + EdOpWeight w = new OcrOpWeight(); + EditSequence eds = new EditSequence(s1, s2, w); + DocumentBuilder builder = new DocumentBuilder("html"); + Element body = builder.addElement("body"); + Element alitab = Aligner.bitext("s1", "s2", s1, s2, w, eds); + builder.addElement(body, alitab); + builder.write(ofile); + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/distance/ArrayEditDistance.java b/ocrevalUAtion/src/main/java/eu/digitisation/distance/ArrayEditDistance.java new file mode 100644 index 00000000..50d84339 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/distance/ArrayEditDistance.java @@ -0,0 +1,166 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.distance; + +/** + * Provides a basic implementations of some popular edit distance methods + * applied to arrays of objects (currently, Levenshtein and indel). This + * implementation can accelerates the computation of distances if, for example, + * text is handled as a sequence of (Integer) word codes. + * + * @version 2011.03.10 + * @param type of content + */ +public class ArrayEditDistance { + + /** + * @return 3-wise minimum. + */ + private static int min(int x, int y, int z) { + return Math.min(x, Math.min(y, z)); + } + + /** + * @param the type of object in the array + * @param first the first string. + * @param second the second string. + * @return the indel distance between first and second. + */ + public static int indel(Type[] first, Type[] second) { + int i, j; + int[][] A = new int[first.length + 1][second.length + 1]; + + // Compute first row + A[0][0] = 0; + for (j = 1; j <= second.length; ++j) { + A[0][j] = A[0][j - 1] + 1; + } + + // Compute other rows + for (i = 1; i <= first.length; ++i) { + A[i][0] = A[i - 1][0] + 1; + for (j = 1; j <= second.length; ++j) { + if (first[i - 1].equals(second[j - 1])) { + A[i][j] = A[i - 1][j - 1]; + } else { + A[i][j] = Math.min(A[i - 1][j] + 1, A[i][j - 1] + 1); + } + } + } + return A[first.length][second.length]; + } + + /** + * @param the type of object in the array + * @param first the first string. + * @param second the second string. + * @return the Levenshtein distance between first and second. + */ + public static int levenshtein(Type[] first, Type[] second) { + int i, j; + int[][] A; + + // intialize + A = new int[first.length + 1][second.length + 1]; + + // Compute first row + A[0][0] = 0; + for (j = 1; j <= second.length; ++j) { + A[0][j] = A[0][j - 1] + 1; + } + + // Compute other rows + for (i = 1; i <= first.length; ++i) { + A[i][0] = A[i - 1][0] + 1; + for (j = 1; j <= second.length; ++j) { + if (first[i - 1] == second[j - 1]) { + A[i][j] = A[i - 1][j - 1]; + } else { + A[i][j] = min(A[i - 1][j] + 1, A[i][j - 1] + 1, + A[i - 1][j - 1] + 1); + } + } + } + return A[first.length][second.length]; + } + + /** + * @param + * @param first the first array. + * @param second the second array. + * @return the Damerau-Levenshtein distance between first and second. + */ + public static int DL(Type[] first, Type[] second) { + int i, j; + int[][] A; + + // intialize + A = new int[3][second.length+ 1]; + + // Compute first row + A[0][0] = 0; + for (j = 1; j <= second.length; ++j) { + A[0][j] = A[0][j - 1] + 1; + } + + // Compute other rows + for (i = 1; i <= first.length; ++i) { + A[i % 3][0] = A[(i - 1) % 3][0] + 1; + for (j = 1; j <= second.length; ++j) { + if (first[i - 1] == second[j - 1]) { + A[i % 3][j] = A[(i - 1) % 3][j - 1]; + } else { + if (i > 1 && j > 1 + && first[i - 1] == second[j - 2] + && first[i - 2] == second[j - 1]) { + A[i % 3][j] = min(A[(i - 1) % 3][j] + 1, + A[i % 3][j - 1] + 1, + A[(i - 2) % 3][j - 2] + 1); + } else { + A[i % 3][j] = min(A[(i - 1) % 3][j] + 1, + A[i % 3][j - 1] + 1, + A[(i - 1) % 3][j - 1] + 1); + } + } + } + } + return A[first.length % 3][second.length]; + } + + /** + * + * @param + * @param first the first array. + * @param second the second array. + * @param type the type of distance to be computed + * @return the distance between first and second (defaults to Levenshtein) + */ + public static int distance(Type[] first, Type[] second, + EditDistanceType type) { + switch (type) { + case INDEL: + return indel(first, second); + case LEVENSHTEIN: + return levenshtein(first, second); + case DAMERAU_LEVENSHTEIN: + return DL(first, second); + default: + return levenshtein(first, second); + } + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/distance/EdOp.java b/ocrevalUAtion/src/main/java/eu/digitisation/distance/EdOp.java new file mode 100644 index 00000000..c98c21e8 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/distance/EdOp.java @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.distance; + +/** + * + * Basic edit operations on a single character + * + * @author R.C.C. + */ +@SuppressWarnings("javadoc") +public enum EdOp { + + KEEP, INSERT, SUBSTITUTE, DELETE; + + @Override + public String toString() { + switch (this) { + case KEEP: + return "K"; + case INSERT: + return "I"; + case SUBSTITUTE: + return "S"; + case DELETE: + return "D"; + default: + return null; + } + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/distance/EdOpWeight.java b/ocrevalUAtion/src/main/java/eu/digitisation/distance/EdOpWeight.java new file mode 100644 index 00000000..7899b452 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/distance/EdOpWeight.java @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +package eu.digitisation.distance; + +/** + * Integer weights for basic edit operations. + * @author R.C.C. + */ +public interface EdOpWeight { + public int sub(char c1, char c2); + public int ins(char c); + public int del(char c); +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/distance/EditDistance.java b/ocrevalUAtion/src/main/java/eu/digitisation/distance/EditDistance.java new file mode 100644 index 00000000..e2c5af02 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/distance/EditDistance.java @@ -0,0 +1,138 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.distance; + +import eu.digitisation.document.TokenArray; +import eu.digitisation.math.MinimalPerfectHash; +import eu.digitisation.text.Text; +import eu.digitisation.text.WordSet; +import java.io.File; + +/** + * Provides linear time implementations of some popular edit distance methods + * operating on strings + * + * @version 2014.01.25 + */ +public class EditDistance { + + /** + * @param s1 the first string. + * @param s2 the second string. + * @param w weights for basic edit operations + * @param chunkLen the length of the chunks analyzed at every step (must be + * strictly greater than 1) + * @return the approximate (linear time) Levenshtein distance between first + * and second. + */ + public static int charDistance(String s1, String s2, EdOpWeight w, int chunkLen) { + EditSequence seq = new EditSequence(s1, s2, w, chunkLen); + return seq.cost(s1, s2, w); + } + + /** + * @param s1 the first string. + * @param s2 the second string. + * @param chunkLen the length of the chunks analyzed at every step (must be + * strictly greater than 1) + * @return the length (number of words) of first, the length (number of + * words) of second, and the approximate (linear time) word-based + * Levenshtein distance between first and second. + */ + public static int[] wordDistance(String s1, String s2, int chunkLen) { + MinimalPerfectHash mph = new MinimalPerfectHash(true); // case sensitive + TokenArray a1 = new TokenArray(mph, s1); + TokenArray a2 = new TokenArray(mph, s2); + EditSequence seq = new EditSequence(a1, a2, chunkLen); + return new int[]{a1.length(), a2.length(), seq.length()}; + } + + /** + * @param s1 the first string. + * @param s2 the second string. + * @param stopwords a set of stop-words + * @param chunkLen the length of the chunks analyzed at every step (must be + * strictly greater than 1) + * @return the length (number of words) of first, the length (number of + * words) of second, and the approximate (linear time) word-based + * Levenshtein distance between first and second. Whenever a stop-word in + * the first string is aligned (substituted) with a different target word in + * the second string or with no string at all (deleted), this difference + * does not contribute to the distance + */ + public static int[] wordDistance(String s1, String s2, + WordSet stopwords, int chunkLen) { + MinimalPerfectHash mph = new MinimalPerfectHash(true); // case sensitive + TokenArray a1 = new TokenArray(mph, s1); + TokenArray a2 = new TokenArray(mph, s2); + EditSequence seq = new EditSequence(a1, a2, chunkLen); + int d = 0; + int n1 = 0; + int n2 = 0; + for (EdOp op : seq.ops) { + String word = a1.wordAt(n1); + if (op != EdOp.KEEP && !stopwords.contains(word)) { + ++d; + } + if (op != EdOp.INSERT) { + ++n1; + } + if (op != EdOp.DELETE) { + ++n2; + } + } + return new int[]{a1.length(), a2.length(), d}; + } + + /** + * + * @param first the first string. + * @param second the second string. + * @param chunkLen the length of the chunks analyzed at every step + * @param type the type of distance to be computed + * @return the distance between first and second (defaults to Levenshtein) + * @throws java.lang.NoSuchMethodException + */ + public static int distance(String first, String second, + int chunkLen, EditDistanceType type) + throws NoSuchMethodException { + switch (type) { + case OCR_CHAR: + EdOpWeight w = new OcrOpWeight(); + return charDistance(first, second, w, chunkLen); + case OCR_WORD: + return wordDistance(first, second, chunkLen)[2]; + default: + throw new java.lang.NoSuchMethodException(type + + " distance still to be implemented"); + + } + } + + public static void main(String[] args) throws Exception { + File f1 = new File(args[0]); + File f2 = new File(args[1]); + int len = Integer.parseInt(args[2]); + String s1 = new Text(f1).toString(); + String s2 = new Text(f2).toString(); + int d = EditDistance.distance(s1, s2, len, EditDistanceType.OCR_CHAR); + System.out.println(d); + d = EditDistance.distance(s1, s2, len, EditDistanceType.OCR_WORD); + System.out.println(d); + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/distance/EditDistanceType.java b/ocrevalUAtion/src/main/java/eu/digitisation/distance/EditDistanceType.java new file mode 100644 index 00000000..2467c984 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/distance/EditDistanceType.java @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.distance; + +/** + * Basic edit distance variants. + * @author R.C.C + */ +public enum EditDistanceType { + + INDEL, LEVENSHTEIN, DAMERAU_LEVENSHTEIN, OCR_CHAR, OCR_WORD; +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/distance/EditSequence.java b/ocrevalUAtion/src/main/java/eu/digitisation/distance/EditSequence.java new file mode 100644 index 00000000..926066b8 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/distance/EditSequence.java @@ -0,0 +1,534 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.distance; + +import eu.digitisation.document.TokenArray; +import eu.digitisation.log.Messages; +import eu.digitisation.math.BiCounter; +import eu.digitisation.text.Text; +import java.io.File; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * An arbitrary length sequence of basic edit operations (keep, insert, + * substitute, delete). + * + * @author R.C.C. + */ +public class EditSequence { + + List ops; // the list of edit operations + int numOps; // the number of non-trivial (KEEP) operations + int length1; // the length of the first string (number of non-INSERT operations in the list) + int length2; // the lenth of the second string (number of non-DELETE operations in the list) + + /** + * Constructs an empty list with an initial capacity of ten. + */ + public EditSequence() { + ops = new ArrayList(); + } + + /** + * Create an EditPAth with the specified initial capacity + * + * @param initialCapacity + */ + public EditSequence(int initialCapacity) { + ops = new ArrayList(initialCapacity); + } + + /** + * + * @param pos a position in the sequence + * @return the basic edit operation in the sequence which is at the + * specified position + */ + public EdOp get(int pos) { + return ops.get(pos); + } + + /** + * Add an operation to the sequence + * + * @param op an edit operation + */ + public final void add(EdOp op) { + ops.add(op); + switch (op) { + case KEEP: + ++length1; + ++length2; + break; + case INSERT: + ++length2; + ++numOps; + break; + case SUBSTITUTE: + ++length1; + ++length2; + ++numOps; + break; + case DELETE: + ++length1; + ++numOps; + break; + } + } + + /** + * Add an operation to the sequence + * + * @param other another sequence of edit operations + */ + public final void append(EditSequence other) { + for (EdOp op : other.ops) { + this.add(op); + } + } + + /** + * Build a new path containing only a prefix of the sequence + * + * @param len the length of the new sequence + * @return the path truncated to the required length + */ + public EditSequence head(int len) { + EditSequence path = new EditSequence(); + for (int n = 0; n < len; ++n) { + path.add(ops.get(n)); + } + return path; + } + + /** + * The size of the list + * + * @return the number of basic edit operations in the sequence + */ + public int size() { + return ops.size(); + } + + /** + * + * @return the number of non-trivial (KEEP) edit operations in this sequence + */ + public int length() { + return numOps; + } + + /** + * The length of the first string + * + * @return the number of edit operations in the sequence involving the first + * string (all but DELETE) + */ + public final int shift1() { + return length1; + } + + /** + * The length of the second string + * + * @return the number of edit operations in the sequence involving the + * second string (all but INSERT) + */ + public final int shift2() { + return length2; + } + + /** + * String representation + * + * @return a string representing the EditSequence + */ + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + for (EdOp op : ops) { + builder.append(op); + } + return builder.toString(); + } + + /** + * Build the EditSequence for a pair of strings + * + * @param first the first string + * @param second the second string + * @param w the weights applied to basic edit operations + */ + public EditSequence(String first, String second, EdOpWeight w) { + int l1; // length of first + int l2; // length of second + int[][] A; // distance table + EditTable B; // edit operations + + // intialize + l1 = first.length(); + l2 = second.length(); + A = new int[2][second.length() + 1]; + B = new EditTable(first.length() + 1, second.length() + 1); + // Compute first row + A[0][0] = 0; + B.set(0, 0, EdOp.KEEP); + for (int j = 1; j <= second.length(); ++j) { + char c2 = second.charAt(j - 1); + A[0][j] = A[0][j - 1] + w.ins(c2); + B.set(0, j, EdOp.INSERT); + } + + // Compute other rows + for (int i = 1; i <= first.length(); ++i) { + char c1 = first.charAt(i - 1); + A[i % 2][0] = A[(i - 1) % 2][0] + w.del(c1); + B.set(i, 0, EdOp.DELETE); + for (int j = 1; j <= second.length(); ++j) { + char c2 = second.charAt(j - 1); + + if (c1 == c2) { + A[i % 2][j] = A[(i - 1) % 2][j - 1]; + B.set(i, j, EdOp.KEEP); + } else { + A[i % 2][j] = Math.min(A[(i - 1) % 2][j - 1] + w.sub(c1, c2), + Math.min(A[i % 2][j - 1] + w.ins(c2), + A[(i - 1) % 2][j] + w.del(c1))); + + if (A[i % 2][j] == A[i % 2][j - 1] + w.ins(c2)) { + B.set(i, j, EdOp.INSERT); + } else if (A[i % 2][j] == A[(i - 1) % 2][j] + w.del(c1)) { + B.set(i, j, EdOp.DELETE); + } else { + B.set(i, j, EdOp.SUBSTITUTE); + } + } + } + } + + // extract sequence of edit operations + int i = B.width - 1; + int j = B.height - 1; + + ops = new ArrayList(); + + while (i > 0 || j > 0) { + EdOp e = null; + try { + e = B.get(i, j); + } catch (Exception ex) { + Messages.severe(i + "," + j); + Messages.severe(B.toString()); + } + switch (e) { + case INSERT: + --j; + break; + case DELETE: + --i; + break; + default: + --i; + --j; + break; + } + add(e); + } + if (i != 0 + || j != 0 + || length1 != first.length() + || length2 != second.length()) { + throw new java.lang.IllegalArgumentException("Unvalid EditTable"); + } else { + Collections.reverse(ops); + } + } + + /** + * Linear-time approximation to the construction of the EditSequence for a + * pair of strings. The complexity gets reduced by splitting the strings + * into smaller, overlapping, chunks. + * + * @param s1 the first string + * @param s2 the second string + * @param w the weights applied to basic edit operations + * @param chunkLen the length of the chunks in which the strings are split + * + */ + public EditSequence(String s1, String s2, EdOpWeight w, int chunkLen) { + int len1 = s1.length(); + int len2 = s2.length(); + + if (chunkLen < 2) { + throw new IllegalArgumentException("chunkLen mut be greater than 1"); + } + + ops = new ArrayList(); + while (shift1() < len1 || shift2() < len2) { + int high1 = Math.min(shift1() + chunkLen, len1); + int high2 = Math.min(shift2() + chunkLen, len2); + String sub1 = s1.substring(shift1(), high1); + String sub2 = s2.substring(shift2(), high2); + EditSequence subseq = new EditSequence(sub1, sub2, w); + EditSequence head = (high1 < len1 || high2 < len2) + ? subseq.head(subseq.size() / 2) + : subseq; + + append(head); + if (len1 > 10 * chunkLen) { + int frac = (100 * length1) / len1; + Messages.info(frac + "% of file processed"); + } + } + } + + /** + * Build the EditSequence for a pair of TokenArrays + * + * @param first the first TokenArray + * @param second the second TokenArray + */ + public EditSequence(TokenArray first, TokenArray second) { + int l1; // length of first + int l2; // length of second + int[][] A; // distance table + EditTable B; // edit operations + + // intialize + l1 = first.length(); + l2 = second.length(); + A = new int[2][second.length() + 1]; + B = new EditTable(first.length() + 1, second.length() + 1); + // Compute first row + A[0][0] = 0; + B.set(0, 0, EdOp.KEEP); + for (int j = 1; j <= second.length(); ++j) { + A[0][j] = A[0][j - 1] + 1; + B.set(0, j, EdOp.INSERT); + } + + // Compute other rows + for (int i = 1; i <= first.length(); ++i) { + int n1 = first.tokenAt(i - 1); + A[i % 2][0] = A[(i - 1) % 2][0] + 1; + B.set(i, 0, EdOp.DELETE); + for (int j = 1; j <= second.length(); ++j) { + int n2 = second.tokenAt(j - 1); + if (n1 == n2) { + A[i % 2][j] = A[(i - 1) % 2][j - 1]; + B.set(i, j, EdOp.KEEP); + } else { + A[i % 2][j] = Math.min(A[(i - 1) % 2][j] + 1, + Math.min(A[i % 2][j - 1] + 1, + A[(i - 1) % 2][j - 1] + 1)); + if (A[i % 2][j] == A[(i - 1) % 2][j] + 1) { + B.set(i, j, EdOp.DELETE); + } else if (A[i % 2][j] == A[i % 2][j - 1] + 1) { + B.set(i, j, EdOp.INSERT); + } else { + B.set(i, j, EdOp.SUBSTITUTE); + } + } + } + } + + // extract sequence of edit operations + int i = B.width - 1; + int j = B.height - 1; + + ops = new ArrayList(); + + while (i > 0 || j > 0) { + EdOp e = B.get(i, j); + switch (e) { + case INSERT: + --j; + break; + case DELETE: + --i; + break; + default: + --i; + --j; + break; + } + add(e); + } + if (i != 0 || j != 0) { + throw new java.lang.IllegalArgumentException("Unvalid EditTable"); + } else { + Collections.reverse(ops); + } + } + + /** + * Linear-time approximation to the construction of the EditSequence for a + * pair of TokenArrays. The complexity gets reduced by splitting the arrays + * into smaller, overlapping, chunks. + * + * @param a1 the first TokenArray + * @param a2 the second TokenArray + * @param chunkLen the length of the chunks in which the arrays are split + * + */ + public EditSequence(TokenArray a1, TokenArray a2, int chunkLen) { + int len1 = a1.length(); + int len2 = a2.length(); + + if (chunkLen < 2) { + throw new IllegalArgumentException("chunkLen mut be greater than 1"); + } + + ops = new ArrayList(); + while (shift1() < len1 || shift2() < len2) { + int high1 = Math.min(shift1() + chunkLen, len1); + int high2 = Math.min(shift2() + chunkLen, len2); + TokenArray sub1 = a1.subArray(shift1(), high1); + TokenArray sub2 = a2.subArray(shift2(), high2); + EditSequence subseq = new EditSequence(sub1, sub2); + + EditSequence head = (high1 < len1 || high2 < len2) + ? subseq.head(subseq.size() / 2) + : subseq; + + append(head); + } + } + + /** + * Extract alignment statistics + * + * @param s1 the source string + * @param s2 the target string + * @return the statistics on the number of edit operations (per character + * and type of operation) + */ + public BiCounter stats(String s1, String s2) { + BiCounter stats = new BiCounter(); + int n1 = 0; + int n2 = 0; + for (EdOp op : ops) { + if (op != EdOp.INSERT) { + stats.inc(s1.charAt(n1), op); + ++n1; + } else { + stats.inc(s2.charAt(n2), EdOp.INSERT); + } + if (op != EdOp.DELETE) { + ++n2; + } + } + return stats; + } + + /** + * Extract alignment statistics + * + * @param s1 the source string + * @param s2 the target string + * @param w weights of basic edit operations + * @return the statistics on the number of edit operations (per character + * and type of operation) + */ + public BiCounter stats(String s1, String s2, EdOpWeight w) { + BiCounter stats = new BiCounter(); + int n1 = 0; + int n2 = 0; + for (EdOp op : ops) { + switch (op) { + case INSERT: + if (w.ins(s2.charAt(n2)) > 0) { + stats.inc(s2.charAt(n2), op); + } // costless insertion is equivalent to neglegible character + ++n2; + break; + case SUBSTITUTE: + if (w.sub(s1.charAt(n1), s2.charAt(n2)) > 0) { + stats.inc(s1.charAt(n1), op); + } else { // costless SUBSTITUTE is equivalent to KEEP + stats.inc(s1.charAt(n1), EdOp.KEEP); + } + ++n1; + ++n2; + break; + case DELETE: + if (w.del(s1.charAt(n1)) > 0) { + stats.inc(s1.charAt(n1), op); + } // costless deletion is equivalent to neglegible character + ++n1; + break; + case KEEP: + stats.inc(s1.charAt(n1), op); + ++n1; + ++n2; + break; + } + } + return stats; + } + + /** + * Compute cost of the transformation + * + * @param s1 the source string + * @param s2 the target string + * @param w the cost of basic edit operations + * @return the added cost of the edit operations (not identical to length + * because some operation may be free) + */ + public int cost(String s1, String s2, EdOpWeight w) { + int added = 0; + int n1 = 0; + int n2 = 0; + for (EdOp op : ops) { + switch (op) { + case INSERT: + added += w.ins(s2.charAt(n2)); + ++n2; + break; + case SUBSTITUTE: + added += w.sub(s1.charAt(n1), s2.charAt(n2)); + ++n1; + ++n2; + break; + case DELETE: + added += w.del(s1.charAt(n1)); + ++n1; + break; + case KEEP: + ++n1; + ++n2; + break; + } + } + return added; + } + + public static void main(String[] args) + throws Exception { + File gtfile = new File(args[0]); + File ocrfile = new File(args[1]); + String gts = new Text(gtfile).toString(); + String ocrs = new Text(ocrfile).toString(); + EdOpWeight w = new OcrOpWeight(); + EditSequence eds = new EditSequence(gts, ocrs, w, 2000); + System.out.println(eds); + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/distance/EditTable.java b/ocrevalUAtion/src/main/java/eu/digitisation/distance/EditTable.java new file mode 100644 index 00000000..1f95b943 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/distance/EditTable.java @@ -0,0 +1,278 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.distance; + +/** + * A compact structure storing the table of edit operations obtained during the + * computation of the edit distance between two sequences a1a2...am and + * b1b2...bn: each cell (i,j) in the table contains the last edit operation in + * the optimal sequence of editions transforming prefix a1a2...ai into prefix + * b1b2...bj. This table supports the retrieval of the full optimal edit + * sequence (equivalent to a shortest path problem). + * + * @author R.C.C. + */ +public class EditTable { + + int width; // table width = max i-value (exclusive) + int height; // table height = max j-value (exclusive) + byte[] bytes; // table content + + /** + * Create an EditTable with width rows and height columns + * + * @param width + * @param height + */ + public EditTable(int width, int height) { + if (Integer.MAX_VALUE / width / height < 4) { + throw new IllegalArgumentException("EditTable is too large"); + } else { + int len = (width / 4) * height + + (width % 4) * (height / 4) + + ((width % 4) * (height % 4)) + 1; // ceiling + this.width = width; + this.height = height; + bytes = new byte[len]; // two bits per operation + } + } + + /** + * Get a specific bit in a byte + * + * @param b + * a byte + * @param position + * the bit position + * @return the byte with that bit set to the specified value + */ + private static boolean getBit(byte b, int position) { + return ((b >> position) & 1) == 1; + } + + /** + * Return a new byte with one bit set to a specific value + * + * @param b + * a byte + * @param position + * the bit position + * @param value + * the value for that bit + * @return a new byte with that bit set to the specified value + */ + private static byte setBit(byte b, int position, boolean value) { + if (value) { + return b |= 1 << position; + } else { + return b |= 0 << position; + } + } + + /** + * Get the bit at that position in the byte array + * + * @param position + * a position in the array + * @return the bit at that position in the byte array + */ + private boolean getBit(int position) { + return getBit(bytes[position / 8], position % 8); + } + + /** + * Set the bit at that position in the byte array to the specified value + * + * @param position + * a position in the array + */ + private void setBit(int position, boolean value) { + bytes[position / 8] = setBit(bytes[position / 8], position % 8, value); + } + + /** + * + * @param i + * x-coordinate + * @param j + * y-coordinate + * @return the edit operation stored at cell (i,j) + * @throws IllegalArgumentException + */ + public EdOp get(int i, int j) { + int bytenum = (i / 4) * height + (i % 4) * (height / 4) + (j / 4) + + ((i % 4) * (height % 4) + (j % 4)) / 4; + int offset = (2 * ((i % 8) * (height % 8) + (j % 8))) % 8; + // int position = 2 * (i * height + j); + + try { + boolean low = getBit(bytes[bytenum], offset); + // boolean low = getBit(position); + boolean high = getBit(bytes[bytenum], offset + 1); + // boolean high = getBit(position + 1); + if (low) { + if (high) { + return EdOp.SUBSTITUTE; + } else { + return EdOp.DELETE; + } + } else { + if (high) { + return EdOp.INSERT; + } else { + return EdOp.KEEP; + } + } + } catch (ArrayIndexOutOfBoundsException ex) { + throw new IllegalArgumentException("Forbiden acces to " + + "cell (" + i + "," + j + + ") in EditTable of size (" + width + "," + height + ")"); + } + } + + /** + * Store an edit operation at cell (i, j) + * + * @param i + * x-coordinate + * @param j + * y-coordinate + * @param op + * the edit operation to be stored + */ + public void set(int i, int j, EdOp op) { + // int position = 2 * (i * height + j); + int bytenum = (i / 4) * height + (i % 4) * (height / 4) + (j / 4) + + ((i % 4) * (height % 4) + (j % 4)) / 4; + int offset = (2 * ((i % 8) * (height % 8) + (j % 8))) % 8; + + boolean low; + boolean high; + + switch (op) { + case SUBSTITUTE: + low = true; + high = true; + break; + case DELETE: + low = true; + high = false; + break; + case INSERT: + low = false; + high = true; + break; + case KEEP: + low = false; + high = false; + break; + default: + low = false; + high = false; + } + try { + bytes[bytenum] = setBit(bytes[bytenum], offset, low); + // setBit(position, low); + bytes[bytenum] = setBit(bytes[bytenum], offset + 1, high); + // setBit(position + 1, high); + } catch (ArrayIndexOutOfBoundsException ex) { + throw new IllegalArgumentException("Forbiden acces to " + + "cell (" + i + "," + j + + ") in EditTable of size (" + width + "," + height + ")"); + } + } + + /** + * + * @return a string representation of the EditTable + */ + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + + for (int i = 0; i < width; ++i) { + for (int j = 0; j < height; ++j) { + EdOp e = get(i, j); + switch (e) { + case KEEP: + builder.append('K'); + break; + case SUBSTITUTE: + builder.append('S'); + break; + case INSERT: + builder.append('I'); + break; + case DELETE: + builder.append('D'); + break; + } + } + builder.append("\n "); + } + + return builder.toString(); + } + + /** + * Build the sequence of edit operations in the path from a the cell at + * (row, column) to the cell at (0,0) + * + * @param row + * the starting row + * @param column + * the starting column + * @return the sequence of edit operations stored in this table which lead + * from the cell at (row, column) to the cell at (0,0) + */ + private EditSequence path(int row, int column) { + EditSequence operations = new EditSequence(Math.max(row, column)); + int i = row; + int j = column; + + while (i > 0 || j > 0) { + EdOp e = get(i, j); + switch (e) { + case INSERT: + --j; + break; + case DELETE: + --i; + break; + default: + --i; + --j; + break; + } + operations.add(e); + } + return (i == 0 && j == 0) ? operations : null; + } + + /** + * Build the sequence of edit operations in the path from a the cell at + * (width, height) to the cell at (0,0) + * + * @return the sequence of edit operations stored in this table which lead + * from the cell at (width - 1, height - 1) to the cell at (0,0) + * @depecated Use new EditSequence(EditTeble) instead + */ + public EditSequence path() { + return path(width - 1, height - 1); + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/distance/OcrOpWeight.java b/ocrevalUAtion/src/main/java/eu/digitisation/distance/OcrOpWeight.java new file mode 100644 index 00000000..10d529a6 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/distance/OcrOpWeight.java @@ -0,0 +1,122 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.distance; + +import eu.digitisation.input.Parameters; + +/** + * Integer weights for basic edit operations. + * + * @author R.C.C. + */ +public class OcrOpWeight implements EdOpWeight { + + // boolean ignoreCase; + // boolean ignoreDiacritics; + boolean ignorePunctuation; + + /** + * + * @param ignoreCase true if case must be ignored + * @param ignoreDiacritics true if diacritics must be ignored + * @param ignorePunctuation true if punctuation must be ignored + */ + public OcrOpWeight(boolean ignorePunctuation) { + // this.ignoreCase = ignoreCase; + // this.ignoreDiacritics = ignoreDiacritics; + this.ignorePunctuation = ignorePunctuation; + } + + public OcrOpWeight(Parameters pars) { + this(pars.ignorePunctuation.getValue()); + } + + /** + * Default constructor creates weights which are case-sensitive, + * diacritics-aware and punctuation-aware. + */ + public OcrOpWeight() { + this(false); + } + + /** + * + * @param c1 the character found in text + * @param c2 the replacing character + * @return the cost of substituting character c1 with c2. Note: whitespace + * must not substitute character which are rather deleted; therefore, such + * cases return a value greater than 2 (standard insertion+deletion). + * Diacritics and case cannot be compared here due to efficiency reasons + * (too slow). + */ + @Override + public int sub(char c1, char c2) { + return (Character.isSpaceChar(c1) ^ Character.isSpaceChar(c2)) ? 4 : 1; + /* + if (Character.isSpaceChar(c1) ^ Character.isSpaceChar(c2)) { + return 4; // replacing whitespace with character is not recommended + } else if (ignoreCase) { + if (ignoreDiacritics) { + String s1 = StringNormalizer.removeDiacritics(String.valueOf(c1)); + String s2 = StringNormalizer.removeDiacritics(String.valueOf(c2)); + return (s1.toLowerCase().equals(s2.toLowerCase())) ? 0 : 1; + } else { + return (Character.toLowerCase(c1) == Character.toLowerCase(c2)) + ? 0 : 1; + } + + } else if (ignoreDiacritics) { // case matters + String s1 = StringNormalizer.removeDiacritics(String.valueOf(c1)); + String s2 = StringNormalizer.removeDiacritics(String.valueOf(c2)); + return (s1.equals(s2)) ? 0 : 1; + } else { + return c1 == c2 ? 0 : 1; + } + */ + } + + /** + * + * @param c a character + * @return the cost of inserting a character c + */ + @Override + public int ins(char c) { + if (ignorePunctuation) { + return (Character.isSpaceChar(c) || Character.isLetterOrDigit(c)) + ? 1 : 0; + } else { + return 1; + } + } + + /** + * + * @param c a character + * @return the cost of removing a character c + */ + @Override + public int del(char c) { + if (ignorePunctuation) { + return (Character.isSpaceChar(c) || Character.isLetterOrDigit(c)) + ? 1 : 0; + } else { + return 1; + } + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/distance/StringEditDistance.java b/ocrevalUAtion/src/main/java/eu/digitisation/distance/StringEditDistance.java new file mode 100644 index 00000000..9cdca62c --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/distance/StringEditDistance.java @@ -0,0 +1,305 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.distance; + +import eu.digitisation.math.BiCounter; +import eu.digitisation.output.ErrorMeasure; +import eu.digitisation.log.Messages; + +/** + * Provides basic implementations of some popular edit distance methods + * operating on strings (currently, Levenshtein and indel) + * + * @version 2011.03.10 + */ +public class StringEditDistance { + + /** + * @return 3-wise minimum. + */ + private static int min(int x, int y, int z) { + return Math.min(x, Math.min(y, z)); + } + + /** + * @param first the first string. + * @param second the second string. + * @return the indel distance between first and second. + */ + public static int indel(String first, String second) { + int i, j; + int[][] A = new int[2][second.length() + 1]; + + // Compute first row + A[0][0] = 0; + for (j = 1; j <= second.length(); ++j) { + A[0][j] = A[0][j - 1] + 1; + } + + // Compute other rows + for (i = 1; i <= first.length(); ++i) { + A[i % 2][0] = A[(i - 1) % 2][0] + 1; + for (j = 1; j <= second.length(); ++j) { + if (first.charAt(i - 1) == second.charAt(j - 1)) { + A[i % 2][j] = A[(i - 1) % 2][j - 1]; + } else { + A[i % 2][j] = Math.min(A[(i - 1) % 2][j] + 1, + A[i % 2][j - 1] + 1); + } + } + } + return A[first.length() % 2][second.length()]; + } + + /** + * @param first the first string. + * @param second the second string. + * @return the Levenshtein distance between first and second. + */ + public static int levenshtein(String first, String second) { + int i, j; + int[][] A; + + // intialize + A = new int[2][second.length() + 1]; + + // Compute first row + A[0][0] = 0; + for (j = 1; j <= second.length(); ++j) { + A[0][j] = A[0][j - 1] + 1; + } + + // Compute other rows + for (i = 1; i <= first.length(); ++i) { + A[i % 2][0] = A[(i - 1) % 2][0] + 1; + for (j = 1; j <= second.length(); ++j) { + if (first.charAt(i - 1) == second.charAt(j - 1)) { + A[i % 2][j] = A[(i - 1) % 2][j - 1]; + } else { + A[i % 2][j] = min(A[(i - 1) % 2][j] + 1, + A[i % 2][j - 1] + 1, + A[(i - 1) % 2][j - 1] + 1); + } + } + } + return A[first.length() % 2][second.length()]; + } + + /** + * @param first the first string. + * @param second the second string. + * @return the Damerau-Levenshtein distance between first and second. + */ + public static int DL(String first, String second) { + int i, j; + int[][] A; + + // intialize + A = new int[3][second.length() + 1]; + + // Compute first row + A[0][0] = 0; + for (j = 1; j <= second.length(); ++j) { + A[0][j] = A[0][j - 1] + 1; + } + + // Compute other rows + for (i = 1; i <= first.length(); ++i) { + A[i % 3][0] = A[(i - 1) % 3][0] + 1; + for (j = 1; j <= second.length(); ++j) { + if (first.charAt(i - 1) == second.charAt(j - 1)) { + A[i % 3][j] = A[(i - 1) % 3][j - 1]; + } else { + if (i > 1 && j > 1 + && first.charAt(i - 1) == second.charAt(j - 2) + && first.charAt(i - 2) == second.charAt(j - 1)) { + A[i % 3][j] = min(A[(i - 1) % 3][j] + 1, + A[i % 3][j - 1] + 1, + A[(i - 2) % 3][j - 2] + 1); + } else { + A[i % 3][j] = min(A[(i - 1) % 3][j] + 1, + A[i % 3][j - 1] + 1, + A[(i - 1) % 3][j - 1] + 1); + } + } + } + } + return A[first.length() % 3][second.length()]; + } + + /** + * + * @param first the first string. + * @param second the second string. + * @param type the type of distance to be computed + * @return the distance between first and second (defaults to Levenshtein) + */ + public static int distance(String first, String second, EditDistanceType type) { + switch (type) { + case INDEL: + return indel(first, second); + case LEVENSHTEIN: + return levenshtein(first, second); + case DAMERAU_LEVENSHTEIN: + return DL(first, second); + default: + return levenshtein(first, second); + } + } + + /** + * Computes the number of edit operations per character + * + * @param first the reference text + * @param second the fuzzy text + * @return a counter with the number of insertions, substitutions and + * deletions for every character + */ + public static BiCounter operations(String first, String second) { + int i, j; + int[][] A; + EditTable B; + BiCounter stats = new BiCounter(); + + // intialize + A = new int[2][second.length() + 1]; + B = new EditTable(first.length() + 1, second.length() + 1); + // Compute first row + A[0][0] = 0; + B.set(0, 0, EdOp.KEEP); + for (j = 1; j <= second.length(); ++j) { + A[0][j] = A[0][j - 1] + 1; + B.set(0, j, EdOp.INSERT); + } + + // Compute other rows + for (i = 1; i <= first.length(); ++i) { + A[i % 2][0] = A[(i - 1) % 2][0] + 1; + B.set(i, 0, EdOp.DELETE); + for (j = 1; j <= second.length(); ++j) { + if (first.charAt(i - 1) == second.charAt(j - 1)) { + A[i % 2][j] = A[(i - 1) % 2][j - 1]; + B.set(i, j, EdOp.KEEP); + } else { + A[i % 2][j] = min(A[(i - 1) % 2][j] + 1, + A[i % 2][j - 1] + 1, + A[(i - 1) % 2][j - 1] + 1); + if (A[i % 2][j] == A[(i - 1) % 2][j] + 1) { + B.set(i, j, EdOp.DELETE); + } else if (A[i % 2][j] == A[i % 2][j - 1] + 1) { + B.set(i, j, EdOp.INSERT); + } else { + B.set(i, j, EdOp.SUBSTITUTE); + } + } + } + } + + i = first.length(); + j = second.length(); + while (i > 0 && j > 0) { + switch (B.get(i, j)) { + case KEEP: + stats.inc(first.charAt(i - 1), EdOp.KEEP); + --i; + --j; + break; + case DELETE: + stats.inc(first.charAt(i - 1), EdOp.DELETE); + --i; + break; + case INSERT: + stats.inc(second.charAt(j - 1), EdOp.INSERT); + --j; + break; + case SUBSTITUTE: + stats.inc(first.charAt(i - 1), EdOp.SUBSTITUTE); + + --i; + --j; + break; + } + } + while (i > 0) { + stats.inc(first.charAt(i - 1), EdOp.DELETE); + --i; + } + while (j > 0) { + stats.inc(second.charAt(j - 1), EdOp.INSERT); + --j; + + } + + return stats; + } + + /** + * Aligns two strings (one to one alignments with substitutions). + * + * @param first the first string. + * @param second the second string. + * @return the mapping between positions. + * @deprecated use Aligner class + */ + public static int[] alignment(String first, String second) { + int i, j; + int[][] A; + + // intialize + A = new int[first.length() + 1][second.length() + 1]; + + // Compute first row + A[0][0] = 0; + for (j = 1; j <= second.length(); ++j) { + A[0][j] = A[0][j - 1] + 1; + } + + // Compute other rows + for (i = 1; i <= first.length(); ++i) { + A[i][0] = A[i - 1][0] + 1; + for (j = 1; j <= second.length(); ++j) { + if (first.charAt(i - 1) == second.charAt(j - 1)) { + A[i][j] = A[i - 1][j - 1]; + } else { + A[i][j] = min(A[i - 1][j] + 1, A[i][j - 1] + 1, + A[i - 1][j - 1] + 1); + } + } + } + + int[] alignments = new int[first.length()]; + java.util.Arrays.fill(alignments, -1); + + i = first.length(); + j = second.length(); + while (i > 0 && j > 0) { + if (first.charAt(i - 1) == second.charAt(j - 1) + || A[i][j] == A[i - 1][j - 1] + 1) { + alignments[--i] = --j; + } else if (A[i][j] == A[i - 1][j] + 1) { + --i; + } else if (A[i][j] == A[i][j - 1] + 1) { + --j; + } else { // remove after debugging + Messages.info(ErrorMeasure.class.getName() + ": Wrong code"); + } + } + + return alignments; + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/distance/WordCompare.java b/ocrevalUAtion/src/main/java/eu/digitisation/distance/WordCompare.java new file mode 100644 index 00000000..6431d2f3 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/distance/WordCompare.java @@ -0,0 +1,149 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.distance; + +import eu.digitisation.document.TokenArray; +import eu.digitisation.math.MinimalPerfectHash; + +/** + * Word alignments between 2 texts (output in text format) + * + * @author R.C.C + */ +public class WordCompare { + + /** + * @return 3-wise minimum. + */ + private static int min(int x, int y, int z) { + return Math.min(x, Math.min(y, z)); + } + + /** + * Compute the table of basic edit operations needed to transform first into + * second + * + * @param first source string + * @param second target string + * @return the table of minimal basic edit operations needed to transform + * first into second + */ + private static EditTable align(TokenArray a1, TokenArray a2) { + + int l1 = a1.length(); // length of first + int l2 = a2.length(); // length of second + int[][] A; // distance table + EditTable B; // edit operations + + // intialize + A = new int[2][l2 + 1]; + B = new EditTable(l1 + 1, l2 + 1); + // Compute first row + A[0][0] = 0; + B.set(0, 0, EdOp.KEEP); + for (int j = 1; j <= l2; ++j) { + A[0][j] = A[0][j - 1] + 1; + B.set(0, j, EdOp.INSERT); + } + + // Compute other rows + for (int i = 1; i <= l1; ++i) { + int n1 = a1.tokenAt(l1 - i); + A[i % 2][0] = A[(i - 1) % 2][0] + 1; + B.set(i, 0, EdOp.DELETE); + for (int j = 1; j <= l2; ++j) { + int n2 = a2.tokenAt(l2 - j); + + if (n1 == n2) { + A[i % 2][j] = A[(i - 1) % 2][j - 1]; + B.set(i, j, EdOp.KEEP); + } else { + A[i % 2][j] = min(A[(i - 1) % 2][j] + 1, + A[i % 2][j - 1] + 1, + A[(i - 1) % 2][j - 1] + 1); + if (A[i % 2][j] == A[(i - 1) % 2][j] + 1) { + B.set(i, j, EdOp.DELETE); + } else if (A[i % 2][j] == A[i % 2][j - 1] + 1) { + B.set(i, j, EdOp.INSERT); + } else { + B.set(i, j, EdOp.SUBSTITUTE); + } + } + } + } + return B; + } + + /** + * Show word-level differences based on a Levenshtein distance + * + * @param first the first text + * @param second the second text + * @return a report on the differences between files + */ + public static String wdiff(String first, String second) { + MinimalPerfectHash mph = new MinimalPerfectHash(false); // case unsensitive + TokenArray a1 = new TokenArray(mph, first); + TokenArray a2 = new TokenArray(mph, second); + EditTable B = align(a1, a2); + StringBuilder builder = new StringBuilder(); + + int l1 = a1.length(); + int l2 = a2.length(); + int i = l1; + int j = l2; + + while (i > 0 && j > 0) { + switch (B.get(i, j)) { + case KEEP: + builder.append(a1.wordAt(l1 - i)).append(" = ") + .append(a2.wordAt(l2 - j)).append('\n'); + --i; + --j; + break; + case DELETE: + builder.append(a1.wordAt(l1 - i)).append(" # []\n"); + --i; + break; + case INSERT: + builder.append("[] # ") + .append(a2.wordAt(l2 - j)).append('\n'); + --j; + break; + case SUBSTITUTE: + builder.append(a1.wordAt(l1 - i)) + .append(" # ").append(a2.wordAt(l2 - j)).append('\n'); + --i; + --j; + break; + } + + } + if (i > 0) { + builder.append(a1.wordAt(l1 - i)).append(" # []\n"); + --i; + } + if (j > 0) { + builder.append("[] # ") + .append(a2.wordAt(l2 - j)).append('\n'); + --j; + } + + return builder.toString(); + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/document/TermFrequencyVector.java b/ocrevalUAtion/src/main/java/eu/digitisation/document/TermFrequencyVector.java new file mode 100644 index 00000000..e2973edc --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/document/TermFrequencyVector.java @@ -0,0 +1,97 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.document; + +import eu.digitisation.math.Counter; +import eu.digitisation.math.MinimalPerfectHash; + +/** + * A term frequency vector stores counts for every word in text. + * + * @author R.C.C. + */ +public class TermFrequencyVector { + + static MinimalPerfectHash mph = new MinimalPerfectHash(); + Counter tf; + + /** + * Create a TermFrequencyVector from a String + * + * @param s + */ + public TermFrequencyVector(String s) { + TokenArray array = new TokenArray(mph, s); + + tf = new Counter(); + for (Integer n : array) { + tf.inc(n); + } + } + + /** + * Compute the distance between two bag of words (order independent + * distance) + * + * @param other another bag of words + * @return the number of differences between this and the other bag of words + */ + public int distance(TermFrequencyVector other) { + int dplus = 0; // excess + int dminus = 0; // fault + for (Integer word : this.tf.keySet()) { + int delta = this.tf.value(word) - other.tf.value(word); + if (delta > 0) { + dplus += delta; + } else { + dminus -= delta; + } + } + for (Integer word : other.tf.keySet()) { + if (!this.tf.containsKey(word)) { + int delta = this.tf.value(word) - other.tf.value(word); + if (delta > 0) { + dplus += delta; + } else { + dminus -= delta; + } + } + } + + return Math.max(dplus, dminus); + } + + /** + * The total number of words + * + * @return the total number of words + */ + public int total() { + return tf.total(); + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + for (Integer code : tf.keyList(Counter.Order.ASCENDING)) { + String s = mph.decode(code); + builder.append(s).append('[').append(tf.get(code)).append("] "); + } + return builder.toString().trim(); + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/document/TokenArray.java b/ocrevalUAtion/src/main/java/eu/digitisation/document/TokenArray.java new file mode 100644 index 00000000..a13c3cce --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/document/TokenArray.java @@ -0,0 +1,155 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.document; + +import eu.digitisation.distance.ArrayEditDistance; +import eu.digitisation.distance.EditDistanceType; +import eu.digitisation.math.MinimalPerfectHash; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +/** + * A TokenArray is a tokenized string: every word is internally stored as an + * integer. The mapping between words and integer codes is shared by all + * TokenArrays created with the same MinimalPerfectHash. + * + * @version 2013.12.10 + */ +public class TokenArray implements Iterable { + + MinimalPerfectHash mph; // the creator mph + Integer[] tokens; // the content + + /** + * Default constructor + * + * @param interpretation the dictionary of codes + * @param tokens the integer representation + */ + TokenArray(MinimalPerfectHash mph, Integer[] tokens) { + this.mph = mph; + this.tokens = tokens; + } + + /** + * Default constructor + * + * @param interpretation the dictionary of codes + * @param tokens the integer representation + * + */ + TokenArray(MinimalPerfectHash mph, List tokens) { + this.mph = mph; + this.tokens = tokens.toArray(new Integer[tokens.size()]); + + } + + /** + * + * @param mph the hashing scheme + * @param s the input string to be tokenized + */ + public TokenArray(MinimalPerfectHash mph, String s) { + this(mph, mph.hashCodes(s)); + } + + /** + * The length of the token array + * + * @return the length of the token array + */ + public int length() { + return tokens.length; + } + + /** + * + * @return the internal representation as an array of integer codes + */ + public Integer[] tokens() { + return tokens; + } + + /** + * Value of token at a given position + * + * @param pos a position in the array + * @return the value associated to this position + */ + public int tokenAt(int pos) { + return tokens[pos]; + } + + /** + * + * @param pos a position in the array + * @return the word or string at this position in the array + */ + public String wordAt(int pos) { + Integer token = tokens[pos]; + return mph.decode(token); + } + + /** + * Create a TokenArray with a range of another TokenArray + * @param fromIndex low endpoint (inclusive) of the subArray + * @param toIndex high endpoint (exclusive) of the subArray + * @return + */ + public TokenArray subArray(int fromIndex, int toIndex) { + List sublist = Arrays.asList(tokens).subList(fromIndex, toIndex); + return new TokenArray(mph, sublist); + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + + for (Integer token : tokens) { + builder.append(mph.decode(token)).append(" "); + } + + return builder.toString().trim(); + } + + /** + * Distance between TokenArrays + * + * @param other another TokenArray + * @param type the distance type + * @return the distance between this and the other TokenArray + */ + public int distance(TokenArray other, EditDistanceType type) { + return ArrayEditDistance.distance(this.tokens, other.tokens, type); + } + + /** + * Return the TokenArray as array + * + * @return the array of tokens + */ + public String array() { + return java.util.Arrays.toString(tokens); + } + + @Override + public Iterator iterator() { + return Arrays.asList(tokens).iterator(); + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/image/Bimage.java b/ocrevalUAtion/src/main/java/eu/digitisation/image/Bimage.java new file mode 100644 index 00000000..b22230ee --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/image/Bimage.java @@ -0,0 +1,340 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.image; + +import eu.digitisation.math.Counter; +import java.awt.BasicStroke; +import java.awt.Color; +import java.awt.Graphics2D; +import java.awt.Image; +import java.awt.Polygon; +import java.awt.color.ColorSpace; +import java.awt.geom.AffineTransform; +import java.awt.image.BufferedImage; +import java.awt.image.ColorConvertOp; +import java.io.IOException; +import java.util.List; +import javax.media.jai.JAI; + +/** + * Extends BufferedImage with some useful operations + * + * @author R.C.C. + */ +public class Bimage extends BufferedImage { + + static int defaultImageType = BufferedImage.TYPE_INT_RGB; + + /** + * Basic constructor + * + * @param width + * @param height + * @param imageType + */ + public Bimage(int width, int height, int imageType) { + super(width, height, imageType); + } + + /** + * Basic constructor + * + * @param width + * @param height + */ + public Bimage(int width, int height) { + super(width, height, defaultImageType); + } + + /** + * Create a BufferedImage from another BufferedImage. Type set to default in + * case of TYPE_CUSTOM (not handled by BufferedImage) . + * + * @param image the source image + */ + public Bimage(BufferedImage image) { + super(image.getWidth(null), image.getHeight(null), + image.getType() == BufferedImage.TYPE_CUSTOM + ? defaultImageType + : image.getType()); + Graphics2D g = createGraphics(); + g.drawImage(image, 0, 0, null); + g.dispose(); + } + + /** + * Create a BufferedImage of the given type from another BufferedImage. + * + * @param image the source image + * @param type the type of BufferedImage + */ + public Bimage(BufferedImage image, int type) { + super(image.getWidth(null), image.getHeight(null), type); + Graphics2D g = createGraphics(); + g.drawImage(image, 0, 0, null); + g.dispose(); + } + + /** + * Create image from file. + * + * @param file the file storing the image + * @throws IOException + * @throws NullPointerException if the file format is unsupported + */ + public Bimage(java.io.File file) throws IOException { + // this(javax.imageio.ImageIO.read(file)); + this(JAI.create("FileLoad", + file.getCanonicalPath()).getAsBufferedImage()); + } + + /** + * Create a scaled image + * + * @param img the source image + * @param scale the scale factor + */ + public Bimage(BufferedImage img, double scale) { + super((int) (scale * img.getWidth()), + (int) (scale * img.getHeight()), + img.getType()); + int hints = java.awt.Image.SCALE_SMOOTH; //scaling algorithm + Image scaled = img.getScaledInstance(this.getWidth(), + this.getHeight(), + hints); + Graphics2D g = createGraphics(); + g.drawImage(scaled, 0, 0, null); + g.dispose(); + } + + /** + * Finds the background (statistical mode of the rgb value for pixels in the + * image) + * + * @return the mode of the color for pixels in this image + */ + private Color background() { + Counter colors = new Counter(); + + for (int x = 0; x < getWidth(); ++x) { + for (int y = 0; y < getHeight(); ++y) { + int rgb = getRGB(x, y); + colors.inc(rgb); + } + } + + Integer mu = colors.maxValue(); + for (Integer n : colors.keySet()) { + if (colors.get(n).equals(mu)) { + return new Color(n); + } + } + return null; + } + + /** + * Create a scaled image + * + * @param scale the scale factor + * @return a scaled image + */ + public Bimage scale(double scale) { + int w = (int) (scale * getWidth()); + int h = (int) (scale * getHeight()); + Bimage scaled = new Bimage(w, h, getType()); + //int hints = java.awt.Image.SCALE_SMOOTH; //scaling algorithm + //Image img = getScaledInstance(w, h, hints); + Graphics2D g = scaled.createGraphics(); + AffineTransform at = new AffineTransform(); + at.scale(scale, scale); + g.drawImage(this, at, null); + g.dispose(); + return scaled; + } + + /** + * Create a rotated image + * + * @param alpha the rotation angle (anticlockwise) + * @return the rotated image + */ + public Bimage rotate(double alpha) { + double cos = Math.cos(alpha); + double sin = Math.abs(Math.sin(alpha)); + int w = (int) Math.floor(getWidth() * cos + getHeight() * sin); + int h = (int) Math.floor(getHeight() * cos + getWidth() * sin); + Bimage rotated = new Bimage(w, h, getType()); + Graphics2D g = (Graphics2D) rotated.getGraphics(); + g.setBackground(background()); + g.clearRect(0, 0, w, h); + if (alpha < 0) { + g.translate(getHeight() * sin, 0); + } else { + g.translate(0, getWidth() * sin); + } + g.rotate(-alpha); + g.drawImage(this, 0, 0, null); + g.dispose(); + return rotated; + + } + + /** + * Create a new image from two layers (with the type of first) + * + * @param first the first source image + * @param second the second source image + */ + public Bimage(BufferedImage first, BufferedImage second) { + super(Math.max(first.getWidth(), second.getWidth()), + Math.max(first.getHeight(), second.getHeight()), + first.getType()); + BufferedImage combined = new BufferedImage(this.getWidth(), + this.getHeight(), + this.getType()); + Graphics2D g = combined.createGraphics(); + g.drawImage(first, 0, 0, null); + g.drawImage(second, 0, 0, null); + g.dispose(); + } + + /** + * Transform image to gray-scale + * + * @return this image as gray-scale image + */ + public Bimage toGrayScale() { + ColorSpace space = ColorSpace.getInstance(ColorSpace.CS_GRAY); + ColorConvertOp operation = new ColorConvertOp(space, null); + return new Bimage(operation.filter(this, null)); + } + + /** + * Transform image to RGB + * + * @return this image as RGB image + */ + public Bimage toRGB() { + Bimage bim = new Bimage(getWidth(), getHeight(), BufferedImage.TYPE_INT_RGB); + Graphics2D g = bim.createGraphics(); + g.drawImage(this, 0, 0, null); + g.dispose(); + return bim; + } + + /** + * Clear the image to white + */ + public void clear() { + Graphics2D g = createGraphics(); + g.setColor(Color.WHITE); + g.fillRect(0, 0, getWidth(), getHeight()); + g.dispose(); + } + + /** + * Add a polygonal frontier to the image + * + * @param p a polygon + * @param color the color of the polygon + * @param stroke the line width in pixels + */ + public void add(Polygon p, Color color, float stroke) { + Graphics2D g = createGraphics(); + g.setColor(color); + g.setStroke(new BasicStroke(stroke)); + g.drawPolygon(p); + g.dispose(); + } + + /** + * Add a dashed polygonal frontier to the image + * + * @param p a polygon + * @param color the color of the polygon + * @param stroke the line width in pixels + * @param pattern the dash pattern, for example, {4f,2f} draws dashes with + * length 4-units and separated 2 units + */ + public void add(Polygon p, Color color, float stroke, float[] pattern) { + Graphics2D g = createGraphics(); + BasicStroke bs = new BasicStroke(stroke, BasicStroke.CAP_BUTT, + BasicStroke.JOIN_MITER, 1, pattern, 0.0f); + g.setColor(color); + g.setStroke(bs); + g.drawPolygon(p); + g.dispose(); + } + + /** + * Add polygonal frontiers to the image + * + * @param polygons list of polygonal regions + * @param color he color of the polygons + * @param stroke the line width in pixels + */ + public void add(List polygons, Color color, float stroke) { + for (Polygon p : polygons) { + add(p, color, stroke); + } + } + + /** + * Add polygonal frontiers to the image + * + * @param polygons an array of polygonal regions + * @param color he color of the polygons + * @param stroke the line width in pixels + * @param pattern the dash pattern, for example, {4f,2f} draws dashes + * 4-pixels long separated by 2 pixels + */ + public void add(Polygon[] polygons, Color color, float stroke, float[] pattern) { + for (Polygon p : polygons) { + add(p, color, stroke, pattern); + } + } + + /** + * Add polygonal frontiers to the image + * + * @param polygons an array of polygonal regions + * @param color he color of the polygons + * @param stroke the line width in pixels + * @param pattern the dash pattern, for example, {4f,2f} draws dashes + * 4-pixels long separated by 2 pixels + */ + public void add(List polygons, Color color, float stroke, float[] pattern) { + for (Polygon p : polygons) { + add(p, color, stroke, pattern); + } + } + + /** + * Write the image to a file + * + * @param file the output file + * @throws java.io.IOException + */ + public void write(java.io.File file) + throws IOException { + Format format = Format.valueOf(file); + JAI.create("filestore", this, + file.getCanonicalPath(), format.toString()); + //javax.imageio.ImageIO.write(this, format, file); + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/image/Display.java b/ocrevalUAtion/src/main/java/eu/digitisation/image/Display.java new file mode 100644 index 00000000..865d072e --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/image/Display.java @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.image; + +import java.awt.Graphics; +import java.awt.image.BufferedImage; +import javax.swing.JComponent; +import javax.swing.JFrame; + +/** + * Simple class to display an image on screen + * + * @author R.C.C. + */ +class ImageComponent extends JComponent { + private static final long serialVersionUID = 1L; + int x; + int y; + int width; + int height; + BufferedImage img = null; + + ImageComponent(BufferedImage img) { + this.img = img; + x = 0; + y = 0; + width = img.getWidth(); + height = img.getHeight(); + } + + ImageComponent(BufferedImage img, int x, int y, int width, int height) { + this.img = img; + this.x = x; + this.y = y; + this.width = width; + this.height = height; + } + + + @Override + public void paint(Graphics g) { + g.drawImage(img, x, y, width, height, this); + // g.finalize(); + } +} + +/** + * Plot an image on screen + * @author R.C.C: + */ +public class Display { + + JFrame window = null; + + public static void draw(BufferedImage img) { + JFrame window = new JFrame(); + window.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); + window.setBounds(0, 0, img.getWidth(), img.getHeight()); + window.getContentPane().add(new ImageComponent(img)); + window.setVisible(true); + } + + public static void draw(BufferedImage img, int width, int height) { + JFrame window = new JFrame(); + window.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); + window.setBounds(0, 0, width, height); + window.getContentPane().add(new ImageComponent(img, 0, 0,width, height)); + window.setVisible(true); + } + +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/image/Format.java b/ocrevalUAtion/src/main/java/eu/digitisation/image/Format.java new file mode 100644 index 00000000..d65e3d33 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/image/Format.java @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.image; + +import java.io.IOException; +import java.util.Locale; + +/** + * Image input formats supported + * + * @author R.C.C + */ +@SuppressWarnings("javadoc") +public enum Format { + + BMP, FlashPix, GIF, JPEG, PNG, PNM, TIFF, WBMP; + + /** + * + * @param file the container file + * @return the Format associated with this file name extension + * @throws IOException + */ + public static Format valueOf(java.io.File file) throws IOException { + String name = file.getName(); + String ext = name + .substring(name.lastIndexOf('.') + 1) + .toLowerCase(Locale.ENGLISH); + if (ext.equals("bpm")) { + return BMP; + } else if (ext.equals("fpx")) { + return FlashPix; + } else if (ext.equals("gif")) { + return GIF; + } else if (ext.equals("jpg") || ext.equals("jpeg")) { + return JPEG; + } else if (ext.equals("png")) { + return PNG; + } else if (ext.equals("pnm")) { + return PNM; + } else if (ext.equals("tif") || ext.equals("tiff")) { + return TIFF; + } else if (ext.equals("wbmp")) { + return WBMP; + } else { + throw new IOException("Unsupported image format " + ext); + } + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/input/Batch.java b/ocrevalUAtion/src/main/java/eu/digitisation/input/Batch.java new file mode 100644 index 00000000..bdea0e41 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/input/Batch.java @@ -0,0 +1,220 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.input; + +import java.io.File; +import java.io.IOException; +import java.io.InvalidObjectException; +import java.nio.file.Path; +import java.util.ArrayList; + +import eu.digitisation.math.Pair; + +/** + * A batch of file pairs to be processed. Files must be in two different folders + * and named unambiguously (a unique one-to-one mapping must be straightforward + * from file names). Alternatively a Batch consisting of a single file pair can + * be also created. + * + * @author R.C.C. + */ +public class Batch { + + int size; + File[] files1; + File[] files2; + + /** + * Create a a batch of file pairs + * + * @param dir1 + * the first directory of files + * @param dir2 + * the second directory of files + * @throws InvalidObjectException + */ + public Batch(File dir1, File dir2) throws InvalidObjectException { + if (dir1.isDirectory()) { + files1 = dir1.listFiles(); + java.util.Arrays.sort(files1); + } else { + files1 = new File[1]; + files1[0] = dir1; + } + if (dir2.isDirectory()) { + files2 = dir2.listFiles(); + java.util.Arrays.sort(files2); + } else { + files2 = new File[1]; + files2[0] = dir2; + } + if (files1.length != files2.length) { + throw new java.io.InvalidObjectException(dir1.getName() + + " and " + dir2.getName() + + " contain a different number of files"); + } else { + size = files1.length; + } + if (!consistent()) { + throw new java.io.InvalidObjectException(dir1.getName() + + " and " + dir2.getName() + + " contain files with inconsistent names"); + } + } + + /** + * + * @param transcriptions + * @param ocrDir + * @throws IOException + * + * @author Paul Vorbach + */ + public Batch(Iterable transcriptions, Path ocrDir) throws IOException { + final ArrayList fs1 = new ArrayList(); + final ArrayList fs2 = new ArrayList(); + + for (final Path transcription : transcriptions) { + fs1.add(transcription.toFile()); + fs2.add(ocrDir.resolve(transcription.getFileName()).toFile()); + } + + size = fs1.size(); + + files1 = new File[size]; + files2 = new File[size]; + + fs1.toArray(files1); + fs2.toArray(files2); + } + + private boolean consistent() { + if (size > 1) { + int low1 = lcp(files1).length(); + int low2 = lcp(files2).length(); + int high1 = lcs(files1).length(); + int high2 = lcs(files2).length(); + for (int n = 0; n < size; ++n) { + String name1 = files1[n].getName(); + String name2 = files2[n].getName(); + String id1 = name1.substring(low1, name1.length() - high1); + String id2 = name2.substring(low2, name2.length() - high2); + if (!id1.equals(id2)) { + return false; + } + } + } + return true; + } + + /** + * + * @return he number of file to be processed + */ + public int size() { + return size; + } + + /** + * + * @param n + * the file number + * @return the n-th File pair + */ + public Pair pair(int n) { + return new Pair(files1[n], files2[n]); + } + + /** + * Common prefix + * + * @param s1 + * a string + * @param s2 + * another string + * @return the common prefix of s1 and s2 + */ + static String prefix(String s1, String s2) { + int limit = Math.min(s1.length(), s2.length()); + int len = 0; + + while (len < limit && s1.charAt(len) == s2.charAt(len)) { + ++len; + } + return s1.substring(0, len); + } + + /** + * Longest common prefix + * + * @param files + * an array of files + * @return the longest common prefix to all filenames + */ + private String lcp(File[] files) { + if (files.length > 0) { + String result = files[0].getName(); + for (int n = 1; n < files.length; ++n) { + result = prefix(result, files[n].getName()); + } + return result; + } else { + return null; + } + } + + /** + * Common suffix + * + * @param s1 + * one word + * @param s2 + * another word + * @return the common suffix to s1 and s2 + */ + static String suffix(String s1, String s2) { + int limit = Math.min(s1.length(), s2.length()); + int len = 0; + + while (len < limit + && s1.charAt(s1.length() - len - 1) == s2.charAt(s2.length() + - len - 1)) { + ++len; + } + return s1.substring(s1.length() - len); + } + + /** + * Longest common suffix + * + * @param files + * an array of files + * @return the longest common suffix to all files + */ + public static String lcs(File[] files) { + if (files.length > 0) { + String result = files[0].getName(); + for (int n = 1; n < files.length; ++n) { + result = suffix(result, files[n].getName()); + } + return result; + } else { + return null; + } + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/input/BooleanSelector.java b/ocrevalUAtion/src/main/java/eu/digitisation/input/BooleanSelector.java new file mode 100644 index 00000000..85a2ff7a --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/input/BooleanSelector.java @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your param) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.input; + +import java.awt.Color; +import java.awt.Dimension; +import java.awt.Font; +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; +import javax.swing.BoxLayout; +import javax.swing.JCheckBox; + +/** + * + * @author R.C.C + */ +public class BooleanSelector extends ParameterSelector { + + private static final long serialVersionUID = 1L; + + JCheckBox box; + + public BooleanSelector(Parameter op, Color forecolor, Color bgcolor) { + super(op, forecolor, bgcolor); + setLayout(new BoxLayout(this, BoxLayout.X_AXIS)); + setPreferredSize(new Dimension(100, 30)); + setBackground(bgcolor); + + box = new JCheckBox(op.name); + box.setFont(new Font("Verdana", Font.BOLD, 12)); + box.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + param.value = box.isSelected(); + } + }); + add(box); + if (op.help != null && op.help.length() > 0) { + add(new Help(op.help, forecolor, bgcolor)); + } + } + +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/input/ExtensionFilter.java b/ocrevalUAtion/src/main/java/eu/digitisation/input/ExtensionFilter.java new file mode 100644 index 00000000..55f78b7d --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/input/ExtensionFilter.java @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.input; + +import java.io.File; +import java.io.FilenameFilter; +import java.util.Locale; + +/** + * + * @author R.C.C. + */ +public class ExtensionFilter implements FilenameFilter { + + private final String ext; + + /** + * Create filter for files with the given extension + * @param ext the extension required + */ + public ExtensionFilter(String ext) { + this.ext = ext.toLowerCase(Locale.ENGLISH); + } + + @Override + public boolean accept(File dir, String name) { + return name.toLowerCase(Locale.ENGLISH).endsWith(ext); + } + +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/input/FileSelector.java b/ocrevalUAtion/src/main/java/eu/digitisation/input/FileSelector.java new file mode 100644 index 00000000..cdcb8851 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/input/FileSelector.java @@ -0,0 +1,215 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.input; + +import eu.digitisation.log.Messages; +import java.awt.BorderLayout; +import java.awt.Color; +import java.awt.Dimension; +import java.awt.Font; +import java.awt.datatransfer.DataFlavor; +import java.awt.datatransfer.UnsupportedFlavorException; +import java.awt.dnd.DnDConstants; +import java.awt.dnd.DropTarget; +import java.awt.dnd.DropTargetDragEvent; +import java.awt.dnd.DropTargetDropEvent; +import java.awt.dnd.DropTargetEvent; +import java.awt.dnd.DropTargetListener; +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import javax.swing.BorderFactory; +import javax.swing.BoxLayout; +import javax.swing.JButton; +import javax.swing.JFileChooser; +import javax.swing.JTextPane; + +/** + * + * @author R.C.C + */ +public class FileSelector extends ParameterSelector { + + static final long serialVersionUID = 1L; + static File dir; // directory opened by default + JTextPane area; // The area to display the filename + JButton choose; // Optional file chooser + + public FileSelector(Parameter op, Color forecolor, Color bgcolor) { + super(op, forecolor, bgcolor); + + setPreferredSize(new Dimension(600, 70)); + setLayout(new BoxLayout(this, BoxLayout.X_AXIS)); + setBorder(BorderFactory.createLineBorder(forecolor, 2)); + + // Drop area + area = new JTextPane(); + area.setFont(new Font("Verdana", Font.PLAIN, 12)); + area.setText("Drop here your " + param.name); + area.setForeground(forecolor); + area.setBackground(bgcolor); + enableDragAndDrop(area); + + // File chooser + choose = new JButton("Or select the file"); + choose.setFont(new Font("Verdana", Font.PLAIN, 10)); + choose.setForeground(forecolor); + choose.setBackground(bgcolor); + choose.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + File file = choose("input_file"); + if (file != null) { + param.setValue(file); + area.setText(file.getName()); + dir = file.getParentFile(); + shade(Color.decode("#B5CC9E")); // green + } + } + }); + + // Put everything togetehr + add(area); + add(choose, BorderLayout.EAST); + } + + /** + * Set the background of panel (excluding choose JButton) + * + * @param color the background color + */ + public void shade(Color color) { + setBackground(color); + area.setBackground(color); + area.setForeground(Color.DARK_GRAY); + } + + /** + * Highlight if not ready + */ + public void checkout() { + if (!ready()) { + shade(Color.decode("#fffacd")); // yellow + } + } + + /** + * Change descriptive text + * + * @param text the text to be displayed + */ + public void setText(String text) { + area.setText(text); + } + + /** + * + * @return the selected input file + */ + public File getFile() { + return param.getValue(); + } + + /** + * + * @return true if a file has been selected and the file exists + */ + public boolean ready() { + File file = getFile(); + return file != null && file.exists(); + } + + /** + * The drag&drop function + * @param area + */ + private void enableDragAndDrop(final JTextPane area) { + DropTarget target; + target = new DropTarget(area, new DropTargetListener() { + + @Override + public void dragEnter(DropTargetDragEvent e) { + } + + @Override + public void dragExit(DropTargetEvent e) { + } + + @Override + public void dragOver(DropTargetDragEvent e) { + } + + @Override + public void dropActionChanged(DropTargetDragEvent e) { + } + + @Override + @SuppressWarnings("unchecked") + public void drop(DropTargetDropEvent e) { + try { + // Accept the drop first! + e.acceptDrop(DnDConstants.ACTION_COPY_OR_MOVE); + if (e.isDataFlavorSupported(DataFlavor.javaFileListFlavor)) { + java.util.List list; + list = (java.util.List) e.getTransferable() + .getTransferData(DataFlavor.javaFileListFlavor); + param.setValue(list.get(0)); + } else if (e.isDataFlavorSupported(DataFlavor.stringFlavor)) { + String name = (String) e.getTransferable() + .getTransferData(DataFlavor.stringFlavor); + param.setValue(new File(new URI(name.trim()))); + } + area.setText(getFile().getName()); + dir = getFile().getParentFile(); + shade(Color.decode("#B5CC9E")); + } catch (URISyntaxException ex) { + Messages.info(FileSelector.class.getName() + ": " + ex); + } catch (IOException ex) { + Messages.info(FileSelector.class.getName() + ": " + ex); + } catch (UnsupportedFlavorException ex) { + Messages.info(FileSelector.class.getName() + ": " + ex); + } + } + }); + } + + /** + * Select file with a file selector (menu) + * + * @param defaultName the default name for the chosen file + * @return + */ + private File choose(String defaultName) { + JFileChooser chooser = new JFileChooser(); + + chooser.setFileSelectionMode(JFileChooser.FILES_AND_DIRECTORIES); + chooser.setDialogTitle("Select input file"); + chooser.setCurrentDirectory(dir); + chooser.setSelectedFile(new File(defaultName)); + int returnVal = chooser.showOpenDialog(FileSelector.this); + if (returnVal == JFileChooser.APPROVE_OPTION) { + return chooser.getSelectedFile(); + } else { + return null; + } + } + +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/input/FileType.java b/ocrevalUAtion/src/main/java/eu/digitisation/input/FileType.java new file mode 100644 index 00000000..9397bb5f --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/input/FileType.java @@ -0,0 +1,164 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.input; + +import eu.digitisation.log.Messages; +import eu.digitisation.text.StringNormalizer; +import eu.digitisation.xml.DocumentParser; +import java.io.File; +import java.io.IOException; +import java.util.Locale; +import java.util.Properties; +import org.w3c.dom.Document; +import org.w3c.dom.Element; + +/** + * Supported input file types + * + * @author R.C.C. + */ +@SuppressWarnings("javadoc") +public enum FileType { + + TEXT, PAGE, FR10, HOCR, ALTO, UNKNOWN; + String tag; + String schemaLocation; // schema URL + + static { + reload(); + } + + public static void reload() { + Properties props = Settings.properties(); + + TEXT.tag = null; // no tag for this type + TEXT.schemaLocation = null; // no schema associated to this type + + PAGE.tag = "PcGts"; + PAGE.schemaLocation = getSchemaLocation(props, "PAGE"); + + FR10.tag = "document"; + FR10.schemaLocation = getSchemaLocation(props, "FR10"); + + ALTO.tag = "alto"; + ALTO.schemaLocation = getSchemaLocation(props, "ALTO"); + + HOCR.tag = "html"; + HOCR.schemaLocation = null; // no schema for this type + } + + /** + * Load the schemaLocation from properties + * + * @param props properties + * @param suffix the schemaLocation suffix (e.g., ALTO, FR10) + * @return the property value + */ + public static String getSchemaLocation(Properties props, String suffix) { + String location = props.getProperty("schemaLocation." + suffix); + return (location == null) ? " " : StringNormalizer.reduceWS(location); + } + + /** + * + * @param locations1 string of URL schema locations separated by spaces + * @param locations2 string of URL schema locations separated by spaces + * @return True if at least one URL is in both locations + */ + private static boolean sameLocation(String locations1, String locations2) { + String[] urls = locations2.split("\\p{Space}+"); + + for (String url : urls) { + if (locations1.contains(url)) { + return true; + } + } + return false; + } + + /** + * + * @param file a file + * @return the FileType of file + */ + public static FileType valueOf(File file) throws SchemaLocationException { + String name = file.getName().toLowerCase(Locale.ENGLISH); + + if (name.endsWith(".txt")) { + return TEXT; + } else if (name.endsWith(".xml")) { + Document doc = DocumentParser.parse(file); + Element root = doc.getDocumentElement(); + String doctype = root.getTagName(); + String location; + + if (root.hasAttribute("xsi:schemaLocation")) { + location = StringNormalizer + .reduceWS(root.getAttribute("xsi:schemaLocation")); + } else if (root.hasAttribute("xsi:noNamespaceSchemaLocation")) { + location = StringNormalizer + .reduceWS(root.getAttribute("xsi:noNamespaceSchemaLocation")); + } else { + location = null; + } + + if (doctype.equals(PAGE.tag)) { + if (sameLocation(location, PAGE.schemaLocation)) { + return PAGE; + } else if (!location.isEmpty()) { + throw new SchemaLocationException(PAGE, location); + } + } else if (doctype.equals(FR10.tag)) { + if (sameLocation(location, FR10.schemaLocation)) { + return FR10; + } else if (!location.isEmpty()) { + throw new SchemaLocationException(FR10, location); + } + } else if (doctype.equals(ALTO.tag)) { + Messages.info(ALTO.schemaLocation); + if (sameLocation(location, ALTO.schemaLocation)) { + return ALTO; + } else if (!location.isEmpty()) { + throw new SchemaLocationException(ALTO, location); + } + } + } else if (name.endsWith(".html")) { + try { + org.jsoup.nodes.Document doc = org.jsoup.Jsoup.parse(file, null); + if (!doc.head().select("meta[name=ocr-system").isEmpty()) { + return HOCR; + } + } catch (IOException ex) { + Messages.info(FileType.class + .getName() + ": " + ex); + } + } + return UNKNOWN; + } + + public static void main(String[] args) { + for (String arg : args) { + try { + File file = new File(arg); + System.out.println(FileType.valueOf(file)); + } catch (SchemaLocationException ex) { + System.out.println(ex); + } + } + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/input/GUI.java b/ocrevalUAtion/src/main/java/eu/digitisation/input/GUI.java new file mode 100644 index 00000000..61d8e0bd --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/input/GUI.java @@ -0,0 +1,347 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.input; + +import eu.digitisation.log.Messages; +import eu.digitisation.ngram.NgramModel; +import eu.digitisation.output.Browser; +import eu.digitisation.output.OutputFileSelector; +import eu.digitisation.output.Report; +import java.awt.BorderLayout; +import java.awt.Color; +import java.awt.Container; +import java.awt.Dimension; +import java.awt.GridLayout; +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; +import java.awt.event.KeyEvent; +import java.io.File; +import java.io.IOException; +import java.io.InvalidObjectException; +import java.nio.charset.Charset; +import javax.swing.Box; +import javax.swing.BoxLayout; +import javax.swing.JButton; +import javax.swing.JCheckBox; +import javax.swing.JFileChooser; +import javax.swing.JFrame; +import javax.swing.JMenu; +import javax.swing.JMenuBar; +import javax.swing.JMenuItem; +import javax.swing.JOptionPane; +import javax.swing.JPanel; +import javax.swing.SwingUtilities; + +/** + * + * @author R.C.C + */ +public class GUI extends JFrame { + + private static final long serialVersionUID = 1L; + private static final Color green = Color.decode("#4C501E"); + private static final Color white = Color.decode("#FAFAFA"); + private static final Color gray = Color.decode("#EEEEEE"); + // Frame components + FileSelector gtselector; + FileSelector ocrselector; + JPanel advanced; + Link info; + JPanel actions; + + /** + * Show a warning message + * + * @param message the text to be displayed + */ + public void warn(String message) { + JOptionPane.showMessageDialog(super.getRootPane(), message, "Error", + JOptionPane.ERROR_MESSAGE); + } + + /** + * Ask for confirmation + * @param message the text to be displayed + * @return true if the option has been confirmed + */ + public boolean confirm(String message) { + return JOptionPane.showConfirmDialog(super.getRootPane(), + message, message, JOptionPane.YES_NO_OPTION) + == JOptionPane.YES_OPTION; + } + + // The unique constructor + public GUI() { + init(); + } + + /** + * Build advanced options panel + * + * @param ignoreCase + * @param ignoreDiacritics + * @param ignorePunctuation + * @param compatibilty + * @param eqfile + * @return + */ + private JPanel advancedOptionsPanel(Parameters pars) { + JPanel panel = new JPanel(new GridLayout(0, 1)); + JPanel subpanel = new JPanel(new GridLayout(0, 2)); + Color fg = getForeground(); + Color bg = getBackground(); + + subpanel.setForeground(fg); + subpanel.setBackground(bg); + subpanel.add(new BooleanSelector(pars.ignoreCase, fg, bg)); + subpanel.add(new BooleanSelector(pars.ignoreDiacritics, fg, bg)); + subpanel.add(new BooleanSelector(pars.ignorePunctuation, fg, bg)); + subpanel.add(new BooleanSelector(pars.compatibility, fg, bg)); + + panel.setForeground(fg); + panel.setBackground(bg); + panel.setVisible(false); + panel.add(subpanel); + panel.add(new FileSelector(pars.swfile, fg, bg)); + panel.add(new FileSelector(pars.eqfile, fg, bg)); + return panel; + } + + /** + * Creates a subpanel with two actions: "show advanced options" & "generate + * report" + * + * @param gui + * @return + */ + private JPanel actionsPanel(final GUI gui, final Parameters pars) { + JPanel panel = new JPanel(); + panel.setLayout(new BoxLayout(panel, BoxLayout.X_AXIS)); + final JCheckBox more = new JCheckBox("Show advanced options"); + more.setForeground(getForeground()); + more.setBackground(Color.LIGHT_GRAY); + more.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + Dimension dframe = gui.getSize(); + Dimension dadvanced = gui.advanced.getPreferredSize(); + if (more.isSelected()) { + gui.setSize(new Dimension(dframe.width, dframe.height + dadvanced.height)); + } else { + gui.setSize(new Dimension(dframe.width, dframe.height - dadvanced.height)); + } + gui.advanced.setVisible(more.isSelected()); + } + }); + + JButton reset = new JButton("Reset"); + reset.setForeground(getForeground()); + reset.setBackground(getBackground()); + reset.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + pars.clear(); + gui.remove(gtselector); + gui.remove(ocrselector); + gui.remove(info); + gui.remove(advanced); + gui.remove(actions); + gui.repaint(); + gui.setVisible(true); + gui.init(); + } + }); + + // Go for it! button with inverted colors + JButton trigger = new JButton("Generate report"); + trigger.setForeground(getBackground()); + trigger.setBackground(getForeground()); + trigger.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + launch(pars); + } + }); + + panel.add(more, BorderLayout.WEST); + panel.add(Box.createHorizontalGlue()); + panel.add(reset, BorderLayout.CENTER); + panel.add(Box.createHorizontalGlue()); + panel.add(trigger, BorderLayout.EAST); + return panel; + } + + private void createReport(Parameters pars) throws WarningException { + try { + Batch batch = new Batch(pars.gtfile.value, pars.ocrfile.value); + Report report = new Report(batch, pars); + File outfile = pars.outfile.getValue(); + report.write(outfile); + Messages.info("Report dumped to " + outfile); + Browser.open(outfile.toURI()); + } catch (InvalidObjectException ex) { + warn(ex.getMessage()); + } catch (SchemaLocationException ex) { + boolean ans = confirm("Unknown schema location:\n" + + ex.getSchemaLocation() + + "\n\nAdd it to the list of valid schemas?"); + if (ans) { + String prop = "schemaLocation." + ex.getFileType(); + String value = ex.getSchemaLocation(); + Settings.addUserProperty(prop, value); + Messages.info(prop + " set to " + + Settings.property(prop)); + } + } catch (IOException ex) { + warn("Input/Output Error"); + } + } + + public void launch(Parameters pars) { + try { + if (ocrselector.ready() && (gtselector.ready())) { + File ocrfile = pars.ocrfile.getValue(); + String name = ocrfile.getName().replaceAll("\\.\\w+", "") + + "_report.html"; + File dir = ocrfile.getParentFile(); + File preselected = new File(name); + OutputFileSelector selector = new OutputFileSelector(); + File outfile = selector.choose(dir, preselected); + pars.outfile.setValue(outfile); + + if (outfile != null) { + createReport(pars); + } + } else { + gtselector.checkout(); + ocrselector.checkout(); + } + } catch (WarningException ex) { + warn(ex.getMessage()); + } + } + + public final void init() { + setDefaultCloseOperation(EXIT_ON_CLOSE); + + // Main container + Container pane = getContentPane(); + // Initialization settings + setForeground(green); + setBackground(gray); + setDefaultCloseOperation(JFrame.DISPOSE_ON_CLOSE); + setLayout(new BoxLayout(pane, BoxLayout.Y_AXIS)); + setLocationRelativeTo(null); + + // Define program parameters: input files + Parameters pars = new Parameters(); + + // Define content + gtselector = new FileSelector(pars.gtfile, getForeground(), white); + ocrselector = new FileSelector(pars.ocrfile, getForeground(), white); + advanced = advancedOptionsPanel(pars); + info = new Link("Info:", "https://sites.google.com/site/textdigitisation/ocrevaluation", getForeground()); + actions = actionsPanel(this, pars); + + // Put all content together + pane.add(gtselector); + pane.add(ocrselector); + pane.add(advanced); + pane.add(info); + pane.add(actions); + + // menu bar + JMenuBar menuBar = new JMenuBar(); + JMenu mainMenu = new JMenu("Main"); + JMenuItem createLanguageModelMenuItem = new JMenuItem("Create Language Model...", KeyEvent.VK_C); + createLanguageModelMenuItem.addActionListener(new ActionListener() { + + @Override + public void actionPerformed(ActionEvent e) { + File inputFile = choose("Choose file to create language model", "sample.txt"); + if (inputFile != null) { + File outputFile = choose("Choose output file", "model.lm"); + if (outputFile != null) { + Object[] possibilities = {"2", "3", "4", "5"}; + String value + = (String) JOptionPane.showInputDialog(null, "Select value vor 'n'", "", + JOptionPane.QUESTION_MESSAGE, null, possibilities, "2"); + if (value != null) { + int n = Integer.parseInt(value); + + NgramModel ngramModel = new NgramModel(n); + Charset encoding = Charset.forName(System.getProperty("file.encoding")); + if (inputFile.isDirectory()) { + File[] files = inputFile.listFiles(); + for (File file : files) { + ngramModel.addWords(file, encoding, false); + } + } else { + ngramModel.addWords(inputFile, encoding, false); + } + ngramModel.save(outputFile); + } + } + } + } + }); + JMenuItem exitMenuItem = new JMenuItem("Exit", KeyEvent.VK_X); + exitMenuItem.addActionListener(new ActionListener() { + + @Override + public void actionPerformed(ActionEvent e) { + System.exit(0); + } + }); + + mainMenu.add(createLanguageModelMenuItem); + mainMenu.addSeparator(); + mainMenu.add(exitMenuItem); + + menuBar.add(mainMenu); + + this.setJMenuBar(menuBar); + + // Show + pack(); + setVisible(true); + } + + private File choose(String title, String defaultName) { + JFileChooser chooser = new JFileChooser(); + + chooser.setFileSelectionMode(JFileChooser.FILES_AND_DIRECTORIES); + chooser.setDialogTitle(title); + chooser.setSelectedFile(new File(defaultName)); + int returnVal = chooser.showOpenDialog(this); + if (returnVal == JFileChooser.APPROVE_OPTION) { + return chooser.getSelectedFile(); + } else { + return null; + } + } + + public static void main(String[] args) { + + SwingUtilities.invokeLater(new Runnable() { + public void run() { + new GUI(); + } + }); + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/input/GUI.java~ b/ocrevalUAtion/src/main/java/eu/digitisation/input/GUI.java~ new file mode 100644 index 00000000..61d8e0bd --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/input/GUI.java~ @@ -0,0 +1,347 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.input; + +import eu.digitisation.log.Messages; +import eu.digitisation.ngram.NgramModel; +import eu.digitisation.output.Browser; +import eu.digitisation.output.OutputFileSelector; +import eu.digitisation.output.Report; +import java.awt.BorderLayout; +import java.awt.Color; +import java.awt.Container; +import java.awt.Dimension; +import java.awt.GridLayout; +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; +import java.awt.event.KeyEvent; +import java.io.File; +import java.io.IOException; +import java.io.InvalidObjectException; +import java.nio.charset.Charset; +import javax.swing.Box; +import javax.swing.BoxLayout; +import javax.swing.JButton; +import javax.swing.JCheckBox; +import javax.swing.JFileChooser; +import javax.swing.JFrame; +import javax.swing.JMenu; +import javax.swing.JMenuBar; +import javax.swing.JMenuItem; +import javax.swing.JOptionPane; +import javax.swing.JPanel; +import javax.swing.SwingUtilities; + +/** + * + * @author R.C.C + */ +public class GUI extends JFrame { + + private static final long serialVersionUID = 1L; + private static final Color green = Color.decode("#4C501E"); + private static final Color white = Color.decode("#FAFAFA"); + private static final Color gray = Color.decode("#EEEEEE"); + // Frame components + FileSelector gtselector; + FileSelector ocrselector; + JPanel advanced; + Link info; + JPanel actions; + + /** + * Show a warning message + * + * @param message the text to be displayed + */ + public void warn(String message) { + JOptionPane.showMessageDialog(super.getRootPane(), message, "Error", + JOptionPane.ERROR_MESSAGE); + } + + /** + * Ask for confirmation + * @param message the text to be displayed + * @return true if the option has been confirmed + */ + public boolean confirm(String message) { + return JOptionPane.showConfirmDialog(super.getRootPane(), + message, message, JOptionPane.YES_NO_OPTION) + == JOptionPane.YES_OPTION; + } + + // The unique constructor + public GUI() { + init(); + } + + /** + * Build advanced options panel + * + * @param ignoreCase + * @param ignoreDiacritics + * @param ignorePunctuation + * @param compatibilty + * @param eqfile + * @return + */ + private JPanel advancedOptionsPanel(Parameters pars) { + JPanel panel = new JPanel(new GridLayout(0, 1)); + JPanel subpanel = new JPanel(new GridLayout(0, 2)); + Color fg = getForeground(); + Color bg = getBackground(); + + subpanel.setForeground(fg); + subpanel.setBackground(bg); + subpanel.add(new BooleanSelector(pars.ignoreCase, fg, bg)); + subpanel.add(new BooleanSelector(pars.ignoreDiacritics, fg, bg)); + subpanel.add(new BooleanSelector(pars.ignorePunctuation, fg, bg)); + subpanel.add(new BooleanSelector(pars.compatibility, fg, bg)); + + panel.setForeground(fg); + panel.setBackground(bg); + panel.setVisible(false); + panel.add(subpanel); + panel.add(new FileSelector(pars.swfile, fg, bg)); + panel.add(new FileSelector(pars.eqfile, fg, bg)); + return panel; + } + + /** + * Creates a subpanel with two actions: "show advanced options" & "generate + * report" + * + * @param gui + * @return + */ + private JPanel actionsPanel(final GUI gui, final Parameters pars) { + JPanel panel = new JPanel(); + panel.setLayout(new BoxLayout(panel, BoxLayout.X_AXIS)); + final JCheckBox more = new JCheckBox("Show advanced options"); + more.setForeground(getForeground()); + more.setBackground(Color.LIGHT_GRAY); + more.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + Dimension dframe = gui.getSize(); + Dimension dadvanced = gui.advanced.getPreferredSize(); + if (more.isSelected()) { + gui.setSize(new Dimension(dframe.width, dframe.height + dadvanced.height)); + } else { + gui.setSize(new Dimension(dframe.width, dframe.height - dadvanced.height)); + } + gui.advanced.setVisible(more.isSelected()); + } + }); + + JButton reset = new JButton("Reset"); + reset.setForeground(getForeground()); + reset.setBackground(getBackground()); + reset.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + pars.clear(); + gui.remove(gtselector); + gui.remove(ocrselector); + gui.remove(info); + gui.remove(advanced); + gui.remove(actions); + gui.repaint(); + gui.setVisible(true); + gui.init(); + } + }); + + // Go for it! button with inverted colors + JButton trigger = new JButton("Generate report"); + trigger.setForeground(getBackground()); + trigger.setBackground(getForeground()); + trigger.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + launch(pars); + } + }); + + panel.add(more, BorderLayout.WEST); + panel.add(Box.createHorizontalGlue()); + panel.add(reset, BorderLayout.CENTER); + panel.add(Box.createHorizontalGlue()); + panel.add(trigger, BorderLayout.EAST); + return panel; + } + + private void createReport(Parameters pars) throws WarningException { + try { + Batch batch = new Batch(pars.gtfile.value, pars.ocrfile.value); + Report report = new Report(batch, pars); + File outfile = pars.outfile.getValue(); + report.write(outfile); + Messages.info("Report dumped to " + outfile); + Browser.open(outfile.toURI()); + } catch (InvalidObjectException ex) { + warn(ex.getMessage()); + } catch (SchemaLocationException ex) { + boolean ans = confirm("Unknown schema location:\n" + + ex.getSchemaLocation() + + "\n\nAdd it to the list of valid schemas?"); + if (ans) { + String prop = "schemaLocation." + ex.getFileType(); + String value = ex.getSchemaLocation(); + Settings.addUserProperty(prop, value); + Messages.info(prop + " set to " + + Settings.property(prop)); + } + } catch (IOException ex) { + warn("Input/Output Error"); + } + } + + public void launch(Parameters pars) { + try { + if (ocrselector.ready() && (gtselector.ready())) { + File ocrfile = pars.ocrfile.getValue(); + String name = ocrfile.getName().replaceAll("\\.\\w+", "") + + "_report.html"; + File dir = ocrfile.getParentFile(); + File preselected = new File(name); + OutputFileSelector selector = new OutputFileSelector(); + File outfile = selector.choose(dir, preselected); + pars.outfile.setValue(outfile); + + if (outfile != null) { + createReport(pars); + } + } else { + gtselector.checkout(); + ocrselector.checkout(); + } + } catch (WarningException ex) { + warn(ex.getMessage()); + } + } + + public final void init() { + setDefaultCloseOperation(EXIT_ON_CLOSE); + + // Main container + Container pane = getContentPane(); + // Initialization settings + setForeground(green); + setBackground(gray); + setDefaultCloseOperation(JFrame.DISPOSE_ON_CLOSE); + setLayout(new BoxLayout(pane, BoxLayout.Y_AXIS)); + setLocationRelativeTo(null); + + // Define program parameters: input files + Parameters pars = new Parameters(); + + // Define content + gtselector = new FileSelector(pars.gtfile, getForeground(), white); + ocrselector = new FileSelector(pars.ocrfile, getForeground(), white); + advanced = advancedOptionsPanel(pars); + info = new Link("Info:", "https://sites.google.com/site/textdigitisation/ocrevaluation", getForeground()); + actions = actionsPanel(this, pars); + + // Put all content together + pane.add(gtselector); + pane.add(ocrselector); + pane.add(advanced); + pane.add(info); + pane.add(actions); + + // menu bar + JMenuBar menuBar = new JMenuBar(); + JMenu mainMenu = new JMenu("Main"); + JMenuItem createLanguageModelMenuItem = new JMenuItem("Create Language Model...", KeyEvent.VK_C); + createLanguageModelMenuItem.addActionListener(new ActionListener() { + + @Override + public void actionPerformed(ActionEvent e) { + File inputFile = choose("Choose file to create language model", "sample.txt"); + if (inputFile != null) { + File outputFile = choose("Choose output file", "model.lm"); + if (outputFile != null) { + Object[] possibilities = {"2", "3", "4", "5"}; + String value + = (String) JOptionPane.showInputDialog(null, "Select value vor 'n'", "", + JOptionPane.QUESTION_MESSAGE, null, possibilities, "2"); + if (value != null) { + int n = Integer.parseInt(value); + + NgramModel ngramModel = new NgramModel(n); + Charset encoding = Charset.forName(System.getProperty("file.encoding")); + if (inputFile.isDirectory()) { + File[] files = inputFile.listFiles(); + for (File file : files) { + ngramModel.addWords(file, encoding, false); + } + } else { + ngramModel.addWords(inputFile, encoding, false); + } + ngramModel.save(outputFile); + } + } + } + } + }); + JMenuItem exitMenuItem = new JMenuItem("Exit", KeyEvent.VK_X); + exitMenuItem.addActionListener(new ActionListener() { + + @Override + public void actionPerformed(ActionEvent e) { + System.exit(0); + } + }); + + mainMenu.add(createLanguageModelMenuItem); + mainMenu.addSeparator(); + mainMenu.add(exitMenuItem); + + menuBar.add(mainMenu); + + this.setJMenuBar(menuBar); + + // Show + pack(); + setVisible(true); + } + + private File choose(String title, String defaultName) { + JFileChooser chooser = new JFileChooser(); + + chooser.setFileSelectionMode(JFileChooser.FILES_AND_DIRECTORIES); + chooser.setDialogTitle(title); + chooser.setSelectedFile(new File(defaultName)); + int returnVal = chooser.showOpenDialog(this); + if (returnVal == JFileChooser.APPROVE_OPTION) { + return chooser.getSelectedFile(); + } else { + return null; + } + } + + public static void main(String[] args) { + + SwingUtilities.invokeLater(new Runnable() { + public void run() { + new GUI(); + } + }); + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/input/GUIHack.java b/ocrevalUAtion/src/main/java/eu/digitisation/input/GUIHack.java new file mode 100644 index 00000000..4e023bbd --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/input/GUIHack.java @@ -0,0 +1,361 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.input; + +import eu.digitisation.log.Messages; +import eu.digitisation.ngram.NgramModel; +import eu.digitisation.ngram.NgramPerplexityEvaluator; +import eu.digitisation.output.Browser; +import eu.digitisation.output.OutputFileSelector; +import eu.digitisation.output.Report; +import eu.digitisation.text.Text; +import java.awt.BorderLayout; +import java.awt.Color; +import java.awt.Container; +import java.awt.Dimension; +import java.awt.GridLayout; +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; +import java.awt.event.KeyEvent; +import java.io.File; +import java.io.IOException; +import java.io.InvalidObjectException; +import java.nio.charset.Charset; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.swing.Box; +import javax.swing.BoxLayout; +import javax.swing.JButton; +import javax.swing.JCheckBox; +import javax.swing.JFileChooser; +import javax.swing.JFrame; +import javax.swing.JMenu; +import javax.swing.JMenuBar; +import javax.swing.JMenuItem; +import javax.swing.JOptionPane; +import javax.swing.JPanel; +import javax.swing.SwingUtilities; + +/** + * + * @author R.C.C + */ +public class GUIHack extends JFrame { + + private static final long serialVersionUID = 1L; + private static final Color green = Color.decode("#4C501E"); + private static final Color white = Color.decode("#FAFAFA"); + private static final Color gray = Color.decode("#EEEEEE"); + // Frame components + FileSelector gtselector; + FileSelector ocrselector; + FileSelector lmselector; + JPanel advanced; + Link info; + JPanel actions; + + /** + * Show a warning message + * + * @param text the text to be displayed + */ + + public void warn(String message) { + JOptionPane.showMessageDialog(super.getRootPane(), message, "Error", + JOptionPane.ERROR_MESSAGE); + } + + /** + * Ask for confirmation + */ + public boolean confirm(String message) { + return JOptionPane.showConfirmDialog(super.getRootPane(), + message, message, JOptionPane.YES_NO_OPTION) + == JOptionPane.YES_OPTION; + } + + // The unique constructor + public GUIHack() { + init(); + } + + /** + * Build advanced options panel + * + * @param ignoreCase + * @param ignoreDiacritics + * @param ignorePunctuation + * @param compatibilty + * @param eqfile + * @return + */ + private JPanel advancedOptionsPanel(Parameters pars) { + JPanel panel = new JPanel(new GridLayout(0, 1)); + JPanel subpanel = new JPanel(new GridLayout(0, 2)); + Color fg = getForeground(); + Color bg = getBackground(); + + subpanel.setForeground(fg); + subpanel.setBackground(bg); + subpanel.add(new BooleanSelector(pars.ignoreCase, fg, bg)); + subpanel.add(new BooleanSelector(pars.ignoreDiacritics, fg, bg)); + subpanel.add(new BooleanSelector(pars.ignorePunctuation, fg, bg)); + subpanel.add(new BooleanSelector(pars.compatibility, fg, bg)); + + panel.setForeground(fg); + panel.setBackground(bg); + panel.setVisible(false); + panel.add(subpanel); + panel.add(new FileSelector(pars.swfile, fg, bg)); + panel.add(new FileSelector(pars.eqfile, fg, bg)); + return panel; + } + + /** + * Creates a subpanel with two actions: "show advanced options" & "generate + * report" + * + * @param gui + * @return + */ + private JPanel actionsPanel(final GUIHack gui, final Parameters pars) { + JPanel panel = new JPanel(); + panel.setLayout(new BoxLayout(panel, BoxLayout.X_AXIS)); + final JCheckBox more = new JCheckBox("Show advanced options"); + more.setForeground(getForeground()); + more.setBackground(Color.LIGHT_GRAY); + more.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + Dimension dframe = gui.getSize(); + Dimension dadvanced = gui.advanced.getPreferredSize(); + if (more.isSelected()) { + gui.setSize(new Dimension(dframe.width, dframe.height + dadvanced.height)); + } else { + gui.setSize(new Dimension(dframe.width, dframe.height - dadvanced.height)); + } + gui.advanced.setVisible(more.isSelected()); + } + }); + + JButton reset = new JButton("Reset"); + reset.setForeground(getForeground()); + reset.setBackground(getBackground()); + reset.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + pars.clear(); + gui.remove(gtselector); + gui.remove(ocrselector); + gui.remove(info); + gui.remove(advanced); + gui.remove(actions); + gui.repaint(); + gui.setVisible(true); + gui.init(); + } + }); + + // Go for it! button with inverted colors + JButton trigger = new JButton("Generate report"); + trigger.setForeground(getBackground()); + trigger.setBackground(getForeground()); + trigger.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + try { + launch(pars); + } catch (SchemaLocationException ex) { + Messages.severe(this.getClass() + ": "+ ex.getMessage()); + } + } + }); + + panel.add(more, BorderLayout.WEST); + panel.add(Box.createHorizontalGlue()); + panel.add(reset, BorderLayout.CENTER); + panel.add(Box.createHorizontalGlue()); + panel.add(trigger, BorderLayout.EAST); + return panel; + } + + public void launch(Parameters pars) throws SchemaLocationException { + try { + if (ocrselector.ready() && (gtselector.ready() || lmselector.ready())) { + File ocrfile = pars.ocrfile.getValue(); + if (gtselector.ready()) { + String name = ocrfile.getName().replaceAll("\\.\\w+", "") + "_report.html"; + File dir = ocrfile.getParentFile(); + File preselected = new File(name); + OutputFileSelector selector = new OutputFileSelector(); + File outfile = selector.choose(dir, preselected); + pars.outfile.setValue(outfile); + + if (outfile != null) { + try { + Batch batch = new Batch(pars.gtfile.value, pars.ocrfile.value); + Report report = new Report(batch, pars); + report.write(outfile); + Messages.info("Report dumped to " + outfile); + Browser.open(outfile.toURI()); + } catch (InvalidObjectException ex) { + warn(ex.getMessage()); + } catch (IOException ex) { + warn("Input/Output Error"); + } + } + } + if (lmselector.ready()) { + Object[] possibilities = {"2", "3", "4", "5"}; + String value + = (String) JOptionPane.showInputDialog(null, "Select contect length", "", + JOptionPane.QUESTION_MESSAGE, null, possibilities, "2"); + if (value != null) { + int contextLength = Integer.parseInt(value); + + NgramPerplexityEvaluator lpc = new NgramPerplexityEvaluator(pars.lmfile.value); + + Text ocr = new Text(ocrfile); + double[] perplexityArray = lpc.calculatePerplexity(ocr.toString(), contextLength); + + LanguageModelEvaluationFrame frame = new LanguageModelEvaluationFrame(); + frame.setInput(ocr.toString(), perplexityArray); + frame.setVisible(true); + } + } + } else { + gtselector.checkout(); + ocrselector.checkout(); + lmselector.checkout(); + } + } catch (WarningException ex) { + warn(ex.getMessage()); + } + } + + public final void init() { + setDefaultCloseOperation(EXIT_ON_CLOSE); + + // Main container + Container pane = getContentPane(); + // Initialization settings + setForeground(green); + setBackground(gray); + setDefaultCloseOperation(JFrame.DISPOSE_ON_CLOSE); + setLayout(new BoxLayout(pane, BoxLayout.Y_AXIS)); + setLocationRelativeTo(null); + + // Define program parameters: input files + Parameters pars = new Parameters(); + + // Define content + gtselector = new FileSelector(pars.gtfile, getForeground(), white); + ocrselector = new FileSelector(pars.ocrfile, getForeground(), white); + lmselector = new FileSelector(pars.lmfile, getForeground(), white); + advanced = advancedOptionsPanel(pars); + info = new Link("Info:", "https://sites.google.com/site/textdigitisation/ocrevaluation", getForeground()); + actions = actionsPanel(this, pars); + + // Put all content together + pane.add(gtselector); + pane.add(ocrselector); + pane.add(lmselector); + pane.add(advanced); + pane.add(info); + pane.add(actions); + + // menu bar + JMenuBar menuBar = new JMenuBar(); + JMenu mainMenu = new JMenu("Main"); + JMenuItem createLanguageModelMenuItem = new JMenuItem("Create Language Model...", KeyEvent.VK_C); + createLanguageModelMenuItem.addActionListener(new ActionListener() { + + @Override + public void actionPerformed(ActionEvent e) { + File inputFile = choose("Choose file to create language model", "sample.txt"); + if (inputFile != null) { + File outputFile = choose("Choose output file", "model.lm"); + if (outputFile != null) { + Object[] possibilities = {"2", "3", "4", "5"}; + String value + = (String) JOptionPane.showInputDialog(null, "Select value vor 'n'", "", + JOptionPane.QUESTION_MESSAGE, null, possibilities, "2"); + if (value != null) { + int n = Integer.parseInt(value); + + NgramModel ngramModel = new NgramModel(n); + Charset encoding = Charset.forName(System.getProperty("file.encoding")); + if (inputFile.isDirectory()) { + File[] files = inputFile.listFiles(); + for (File file : files) { + ngramModel.addWords(file, encoding, false); + } + } else { + ngramModel.addWords(inputFile, encoding, false); + } + ngramModel.save(outputFile); + } + } + } + } + }); + JMenuItem exitMenuItem = new JMenuItem("Exit", KeyEvent.VK_X); + exitMenuItem.addActionListener(new ActionListener() { + + @Override + public void actionPerformed(ActionEvent e) { + System.exit(0); + } + }); + + mainMenu.add(createLanguageModelMenuItem); + mainMenu.addSeparator(); + mainMenu.add(exitMenuItem); + + menuBar.add(mainMenu); + + this.setJMenuBar(menuBar); + + // Show + pack(); + setVisible(true); + } + + private File choose(String title, String defaultName) { + JFileChooser chooser = new JFileChooser(); + + chooser.setFileSelectionMode(JFileChooser.FILES_AND_DIRECTORIES); + chooser.setDialogTitle(title); + chooser.setSelectedFile(new File(defaultName)); + int returnVal = chooser.showOpenDialog(this); + if (returnVal == JFileChooser.APPROVE_OPTION) { + return chooser.getSelectedFile(); + } else { + return null; + } + } + + public static void main(String[] args) { + + SwingUtilities.invokeLater(new Runnable() { + public void run() { + new GUI(); + } + }); + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/input/Help.java b/ocrevalUAtion/src/main/java/eu/digitisation/input/Help.java new file mode 100644 index 00000000..13f4022f --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/input/Help.java @@ -0,0 +1,109 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.input; + +import eu.digitisation.log.Messages; +import eu.digitisation.output.Browser; +import java.awt.Color; +import java.awt.Container; +import java.awt.Dimension; +import java.awt.Graphics; +import java.awt.Shape; +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; +import java.awt.geom.Ellipse2D; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.swing.JButton; +import javax.swing.JOptionPane; + +/** + * Help text or URL for additional information (URL must start with http:) + * + * @author R.C.C. + */ +public class Help extends JButton { + + private static final long serialVersionUID = 1L; + String text; // help text + + /** + * Default constructor + * + * @param helpText the help text or URL + * @param forecolor foreground color + * @param bgcolor background color + */ + public Help(String helpText, Color forecolor, Color bgcolor) { + super("?"); + setPreferredSize(new Dimension(10, 10)); + setForeground(forecolor); + setBackground(bgcolor); + setContentAreaFilled(false); + + this.text = helpText; + + addActionListener(new ActionListener() { + Container container = getParent(); + + @Override + public void actionPerformed(ActionEvent e) { + if (text.startsWith("http:")) { + try { + Browser.open(new URI(text)); + } catch (URISyntaxException ex) { + Messages.severe(Help.class.getName() + ex); + } + } else { + JOptionPane.showMessageDialog(getParent(), text); + } + } + }); + } + + // Artwork + @Override + protected void paintComponent(Graphics g) { + if (getModel().isArmed()) { + g.setColor(Color.lightGray); + } else { + g.setColor(getBackground()); + } + g.fillOval(7, 0, getSize().width - 16, getSize().height - 1); + + super.paintComponent(g); + } + + @Override + protected void paintBorder(Graphics g) { + g.setColor(getForeground()); + g.drawOval(7, 0, getSize().width - 16, getSize().height - 1); + } + Shape shape; + + @Override + public boolean contains(int x, int y) { + if (shape == null + || !shape.getBounds().equals(getBounds())) { + shape = new Ellipse2D.Float(7, 0, getWidth() - 7, getHeight()); + } + return shape.contains(x, y); + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/input/LanguageModelEvaluationFrame.java b/ocrevalUAtion/src/main/java/eu/digitisation/input/LanguageModelEvaluationFrame.java new file mode 100644 index 00000000..7241e835 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/input/LanguageModelEvaluationFrame.java @@ -0,0 +1,238 @@ +package eu.digitisation.input; + +import eu.digitisation.text.Text; +import java.awt.Color; +import java.awt.EventQueue; +import java.awt.Font; +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import javax.swing.GroupLayout; +import javax.swing.GroupLayout.Alignment; +import javax.swing.JFrame; +import javax.swing.JOptionPane; +import javax.swing.JPanel; +import javax.swing.JScrollPane; +import javax.swing.JSlider; +import javax.swing.JTextField; +import javax.swing.JTextPane; +import javax.swing.LayoutStyle.ComponentPlacement; +import javax.swing.border.EmptyBorder; +import javax.swing.event.ChangeEvent; +import javax.swing.event.ChangeListener; +import javax.swing.text.SimpleAttributeSet; +import javax.swing.text.StyleConstants; +import javax.swing.text.StyledDocument; + +public class LanguageModelEvaluationFrame extends JFrame { + + /** + * */ + private static final long serialVersionUID = 4895806099667768081L; + + private JPanel contentPane; + private JTextField thresholdTextField; + private JSlider thresholdSlider; + private JTextPane textPane; + + /** + * double array containing the perplexity values for every character. + */ + private double[] perplexityArray; + + /** + * text style applied when the threshold value is exceeded. + */ + private static final SimpleAttributeSet thresholdExceededStyle = new SimpleAttributeSet(); + /** + * default text style. + */ + private static final SimpleAttributeSet defaultStyle = new SimpleAttributeSet(); + + static { + StyleConstants.setForeground(thresholdExceededStyle, Color.red); + StyleConstants.setForeground(defaultStyle, Color.black); + } + + /** + * Create the frame. + */ + public LanguageModelEvaluationFrame() { + init(); + } + + /** + * GUI initialization. + */ + private void init() { + setBounds(100, 100, 473, 347); + contentPane = new JPanel(); + contentPane.setBorder(new EmptyBorder(5, 5, 5, 5)); + setContentPane(contentPane); + + JPanel panel = new JPanel(); + + JScrollPane scrollPane = new JScrollPane(); + GroupLayout gl_contentPane = new GroupLayout(contentPane); + gl_contentPane.setHorizontalGroup(gl_contentPane.createParallelGroup(Alignment.LEADING) + .addComponent(panel, GroupLayout.DEFAULT_SIZE, 447, Short.MAX_VALUE) + .addComponent(scrollPane, GroupLayout.DEFAULT_SIZE, 447, Short.MAX_VALUE)); + gl_contentPane.setVerticalGroup(gl_contentPane.createParallelGroup(Alignment.LEADING).addGroup( + gl_contentPane + .createSequentialGroup() + .addComponent(panel, GroupLayout.PREFERRED_SIZE, GroupLayout.DEFAULT_SIZE, + GroupLayout.PREFERRED_SIZE).addPreferredGap(ComponentPlacement.RELATED) + .addComponent(scrollPane, GroupLayout.DEFAULT_SIZE, 240, Short.MAX_VALUE))); + + textPane = new JTextPane(); + textPane.setFont(new Font("Tahoma", Font.PLAIN, 16)); + scrollPane.setViewportView(textPane); + + thresholdTextField = new JTextField(); + thresholdTextField.setEditable(false); + thresholdTextField.setFont(new Font("Tahoma", Font.BOLD, 16)); + thresholdTextField.setText("-1"); + thresholdTextField.setColumns(10); + + thresholdSlider = new JSlider(); + thresholdSlider.setValue(-1); + thresholdSlider.setMaximum(-1); + thresholdSlider.setMinimum(-50); + thresholdSlider.addChangeListener(new ChangeListener() { + + @Override + public void stateChanged(ChangeEvent e) { + if (!thresholdSlider.getValueIsAdjusting()) { + thresholdTextField.setText((double) thresholdSlider.getValue() + ""); + update(true); + } + } + }); + GroupLayout gl_panel = new GroupLayout(panel); + gl_panel.setHorizontalGroup(gl_panel.createParallelGroup(Alignment.TRAILING).addGroup( + gl_panel.createSequentialGroup() + .addComponent(thresholdSlider, GroupLayout.DEFAULT_SIZE, 357, Short.MAX_VALUE) + .addPreferredGap(ComponentPlacement.RELATED) + .addComponent(thresholdTextField, GroupLayout.PREFERRED_SIZE, 74, GroupLayout.PREFERRED_SIZE) + .addContainerGap())); + gl_panel.setVerticalGroup(gl_panel.createParallelGroup(Alignment.TRAILING).addGroup( + gl_panel.createSequentialGroup() + .addContainerGap() + .addGroup( + gl_panel.createParallelGroup(Alignment.TRAILING) + .addComponent(thresholdSlider, Alignment.LEADING, GroupLayout.DEFAULT_SIZE, 31, + Short.MAX_VALUE) + .addComponent(thresholdTextField, Alignment.LEADING, GroupLayout.DEFAULT_SIZE, + 31, Short.MAX_VALUE)).addContainerGap())); + panel.setLayout(gl_panel); + contentPane.setLayout(gl_contentPane); + + } + + /** + * update the evaluation results. + */ + private void update(boolean thresholdMode) { + Double threshold = 0.0; + try { + threshold = Double.parseDouble(thresholdTextField.getText()); + StyledDocument document = textPane.getStyledDocument(); + + document.setCharacterAttributes(0, document.getLength(), defaultStyle, true); + if (thresholdMode) { + for (int i = 0; i < perplexityArray.length; i++) { + if (perplexityArray[i] < threshold) { + document.setCharacterAttributes(i, 1, thresholdExceededStyle, true); + } + } + } else { + double max = 0; + + for (int i = 0; i < perplexityArray.length; i++) { + double value = perplexityArray[i]; + if (!Double.isInfinite(value)) { + if (Math.abs(value) > Math.abs(max)) { + max = value; + } + } + } + + List colors = getColorBands(Color.red, 11); + + for (int i = 0; i < perplexityArray.length; i++) { + SimpleAttributeSet style = new SimpleAttributeSet(); + double value = perplexityArray[i]; + if (Double.isInfinite(value)) { + StyleConstants.setForeground(style, Color.red); + } else { + int color = 10 - (int) ((value / max) * 10); + StyleConstants.setForeground(style, colors.get(color)); + } + document.setCharacterAttributes(i, 1, style, true); + } + } + + } catch (NumberFormatException nfe) { + JOptionPane.showMessageDialog(this, "Unable to parse value '" + thresholdTextField.getText() + "'", + "Error", JOptionPane.ERROR_MESSAGE); + } + } + + public void setInput(String textToEvaluate, double[] perplexityArray) { + this.perplexityArray = perplexityArray; + textPane.setText(textToEvaluate); + update(false); + } + + public List getColorBands(Color color, int bands) { + + List colorBands = new ArrayList(bands); + for (int index = 0; index < bands; index++) { + colorBands.add(darken(color, (double) index / (double) bands)); + } + return colorBands; + + } + + public static Color darken(Color color, double fraction) { + + int red = (int) Math.round(Math.max(0, color.getRed() - 255 * fraction)); + int green = (int) Math.round(Math.max(0, color.getGreen() - 255 * fraction)); + int blue = (int) Math.round(Math.max(0, color.getBlue() - 255 * fraction)); + + int alpha = color.getAlpha(); + + return new Color(red, green, blue, alpha); + + } + + /** + * Launch the application. + */ + public static void main(final String[] args) { + + EventQueue.invokeLater(new Runnable() { + public void run() { + try { + Text ocr = new Text(new File(args[0])); + final double[] perplexityArray + = new double[]{0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.0, 0.1, 0.2, 0.3, 0.4, + 0.5, 0.6, 0.7, 0.8, 0.9, 0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.0, + 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, + 0.7, 0.8, 0.9, 0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.0, 0.1, 0.2, + 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, + 0.9, 0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.0, 0.1, 0.2, 0.3, 0.4, + 0.5, 0.6, 0.7, 0.8, 0.9, 0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.0, + 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9}; + + LanguageModelEvaluationFrame frame = new LanguageModelEvaluationFrame(); + frame.setInput(ocr.toString(), perplexityArray); + frame.setVisible(true); + + } catch (Exception e) { + e.printStackTrace(); + } + } + }); + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/input/Link.java b/ocrevalUAtion/src/main/java/eu/digitisation/input/Link.java new file mode 100644 index 00000000..46ea7280 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/input/Link.java @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.input; + + +import eu.digitisation.log.Messages; +import eu.digitisation.output.Browser; +import java.awt.Color; +import java.awt.Cursor; +import java.awt.Dimension; +import java.awt.Font; +import java.awt.event.MouseAdapter; +import java.awt.event.MouseEvent; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.swing.JLabel; +import javax.swing.JPanel; + +/** + * + * @author R.C.C + */ +public class Link extends JPanel { + private static final long serialVersionUID = 1L; + JLabel link; + /** + * Basic constructor + * @param title the text to be shown + * @param url the linked URL + * @param color the color of the link + */ + public Link(final String title, final String url, Color color) { + setPreferredSize(new Dimension(600,30)); + link = new JLabel(); + link.setFont(new Font("Verdana", Font.PLAIN, 12)); + link.setAlignmentX(LEFT_ALIGNMENT); + link.setText("" + title + + "" + url + + ""); + link.setCursor(new Cursor(Cursor.HAND_CURSOR)); + link.addMouseListener(new MouseAdapter() { + @Override + public void mouseClicked(MouseEvent e) { + try { + Browser.open(new URI(url)); + } catch (URISyntaxException ex) { + Messages.severe(Link.class.getName() + ex); + } + } + }); + add(link); + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/input/Parameter.java b/ocrevalUAtion/src/main/java/eu/digitisation/input/Parameter.java new file mode 100644 index 00000000..7af017ee --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/input/Parameter.java @@ -0,0 +1,107 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.input; + +/** + * A program parameter with a value, a name (a short description) and, + * optionally, a help text or URL providing a longer description. + * + * @author R.C.C. + * @param the type of parameter (Boolean, File) + */ +public class Parameter { + + String name; + Type value; + String help; // text help or URL + + /** + * Crete a Parameter with the given name (and null value) + * + * @param name the parameter's name + */ + Parameter(String name) { + this.name = name; + } + + /** + * Create Parameter with the given name and set this parameter's help text + * and URL with additional help + * + * @param name the parameter's name + * @param value this parameter's value + * @param help help text or URL for this parameter + */ + Parameter(String name, Type value, String help) { + this.name = name; + this.value = value; + this.help = help; + } + + /** + * Set this parameter's value + * + * @param value the parameter's value + */ + public void setValue(Type value) { + this.value = value; + } + + /** + * Get this parameter's value + * + * @return the parameter's value + */ + public Type getValue() { + return value; + } + + /** + * Get the parameter value type (Boolean, File, Integer,...) + * + * @return + */ + public Class getType() { + return value.getClass(); + } + + /** + * + * @returna short description of the parameter + */ + public String getName() { + return name; + } + + /** + * + * @return the help text for this parameter + */ + public String getHelp() { + return help; + } + + /** + * + * @return a string name:value + */ + @Override + public String toString() { + return name + ":" + value; + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/input/ParameterSelector.java b/ocrevalUAtion/src/main/java/eu/digitisation/input/ParameterSelector.java new file mode 100644 index 00000000..615caf74 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/input/ParameterSelector.java @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.input; + +import java.awt.Color; +import javax.swing.JPanel; + +/** + * + * @author R.C.C + * @param the type of parameter (Boolean, File, ...) + */ +public abstract class ParameterSelector extends JPanel { + private static final long serialVersionUID = 1L; + + Parameter param; + + public ParameterSelector(Parameter param, Color forecolor, Color backcolor) { + this.param = param; + setForeground(forecolor); + setBackground(backcolor); + } + + public Parameter getOption() { + return param; + } + +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/input/Parameters.java b/ocrevalUAtion/src/main/java/eu/digitisation/input/Parameters.java new file mode 100644 index 00000000..8e56f85c --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/input/Parameters.java @@ -0,0 +1,76 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.input; + +import java.io.File; + +/** + * Stores all the input parameters used by the program + * + * @author R.C.C. + */ +public class Parameters { + + private static final long serialVersionUID = 1L; + // Define program parameters: input files + public final Parameter gtfile; + public final Parameter ocrfile; + public final Parameter eqfile; // equivalences + public final Parameter swfile; // stop words + public final Parameter lmfile; // language model + public final Parameter outfile; + // Define program parameters: boolean options + public final Parameter ignoreCase; + public final Parameter ignoreDiacritics; + public final Parameter ignorePunctuation; + public final Parameter compatibility; + // Define program parameters: String options + public final Parameter encoding; + // Set verbosity during debugging (unused) + public final Parameter verbose; + + public Parameters() { + gtfile = new Parameter("ground-truth file"); + ocrfile = new Parameter("OCR file"); + eqfile = new Parameter("Unicode equivalences file"); + swfile = new Parameter("stop-words file"); + lmfile = new Parameter("Language model file"); + outfile = new Parameter("output file"); + ignoreCase = new Parameter("Ignore case", false, ""); + ignoreDiacritics = new Parameter("Ignore diacritics", false, ""); + ignorePunctuation = new Parameter("Ignore punctuation", false, ""); + compatibility = new Parameter("Unicode compatibility characters", false, + "http://unicode.org/reports/tr15/#Canon_Compat_Equivalence"); + encoding = new Parameter("Text file encoding"); + verbose = new Parameter("Verbose", false, ""); + } + + public void clear() { + gtfile.setValue(null); + ocrfile.setValue(null); + eqfile.setValue(null); + swfile.setValue(null); + lmfile.setValue(null); + outfile.setValue(null); + ignoreCase.setValue(null); + ignoreDiacritics.setValue(null); + ignorePunctuation.setValue(null); + compatibility.setValue(null); + encoding.setValue(null); + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/input/SchemaLocationException.java b/ocrevalUAtion/src/main/java/eu/digitisation/input/SchemaLocationException.java new file mode 100644 index 00000000..0b251f23 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/input/SchemaLocationException.java @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.input; + +import java.io.IOException; + +/** + * An exception raised because the schema for the XML file is unknown + * @author R.C.C + */ + +public class SchemaLocationException extends IOException { + FileType type; + String schemaLocation; + + SchemaLocationException(FileType type, String schemaLocation) { + this.type = type; + this.schemaLocation = schemaLocation; + } + + public FileType getFileType() { + return type; + } + + public String getSchemaLocation() { + return schemaLocation; + } + + @Override + public String toString() { + return "Unknown schema location " + schemaLocation + + " for file type " + type; + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/input/Settings.java b/ocrevalUAtion/src/main/java/eu/digitisation/input/Settings.java new file mode 100644 index 00000000..ac7fe961 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/input/Settings.java @@ -0,0 +1,156 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.input; + +import eu.digitisation.log.Messages; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Properties; + +/** + * Start-up actions: load default and user properties (user-defined values + * overwrite defaults). + * + * @author R.C.C. + */ +public class Settings { + + private static Properties props = new Properties(); + + /** + * Get application directory + */ + private static File appDir() { + try { + URI uri = Messages.class.getProtectionDomain() + .getCodeSource().getLocation().toURI(); + File dir = new File(uri.getPath()).getParentFile(); + + Messages.info("Application folder is " + dir); + return dir; + } catch (URISyntaxException ex) { + Messages.severe(Settings.class.getName() + ": " + ex); + } + return null; + } + + static { + try { + InputStream in; + // Read defaults + Properties defaults = new Properties(); + in = Settings.class.getResourceAsStream("/defaultProperties.xml"); + if (in != null) { + defaults.loadFromXML(in); + in.close(); + props = new Properties(defaults); + } + + // Add user properties (may overwrite defaults) + File file = new File(appDir(), "userProperties.xml"); + if (file.exists()) { + in = new FileInputStream(file); + props.loadFromXML(in); + Messages.info("Read properties from " + file); + in.close(); + } else { + in = Settings.class.getResourceAsStream("/userProperties.xml"); + if (in != null) { + defaults.loadFromXML(in); + Messages.info("Read properties from " + file); + in.close(); + props = new Properties(defaults); + } else { + Messages.info("No properties were defined by user"); + } + } + } catch (IOException ex) { + Messages.severe(Settings.class.getName() + ": " + ex); + } + } + + /** + * @return the properties defined at startup (user-defined overwrite + * defaults). + */ + public static Properties properties() { + return props; + } + + /** + * + * @param key a property name + * @return the property with the specified key as defined by the user, and + * otherwise, its default value ( (if the default is not defined, then the + * method returns null). + */ + public static String property(String key) { + return props.getProperty(key); + } + + /** + * Add a new value to property + * + * @param type + * @param schemaLocation + */ + public static void addUserProperty2(FileType type, String schemaLocation) { + String prop = props.getProperty("schemaLocation." + type); + String value = props.getProperty(prop); + props.setProperty(prop, value + " " + schemaLocation); + saveToFile(); + } + + /** + * Add a new value to property + * + * @param type + * @param schemaLocation + */ + public static void addUserProperty(String prop, String value) { + String currentValue = props.getProperty(prop); + if (currentValue == null) { + props.setProperty(prop, value); + } else { + props.setProperty(prop, currentValue + " " + value); + } + saveToFile(); + } + + /** + * Save properties to XML file (userProperties.xml) + */ + private static void saveToFile() { + try { + File file = new File(appDir(), "userProperties.xml"); + OutputStream os = new FileOutputStream(file); + props.storeToXML(os, null); + os.close(); + Messages.info("Created new file: " + file.getAbsolutePath()); + FileType.reload(); + } catch (IOException ex) { + Messages.severe(Settings.class.getName() + ": " + ex); + } + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/input/WarningException.java b/ocrevalUAtion/src/main/java/eu/digitisation/input/WarningException.java new file mode 100644 index 00000000..8cb1073c --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/input/WarningException.java @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.input; + +/** + * Exceptions which only generate a warning and waits for user reaction + * @author R.C.C. + */ +public class WarningException extends Exception { + private static final long serialVersionUID = 1L; + + /** + * Default constructor + * @param message + */ + public WarningException(String message) { + super(message); + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/langutils/TermFrequency.java b/ocrevalUAtion/src/main/java/eu/digitisation/langutils/TermFrequency.java new file mode 100644 index 00000000..e943599d --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/langutils/TermFrequency.java @@ -0,0 +1,241 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.langutils; + +import eu.digitisation.input.SchemaLocationException; +import eu.digitisation.input.WarningException; +import eu.digitisation.log.Messages; +import eu.digitisation.math.Counter; +import eu.digitisation.text.CharFilter; +import eu.digitisation.text.StringNormalizer; +import eu.digitisation.text.Text; +import eu.digitisation.text.WordScanner; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Compute term frequencies in a collection + * + * @author R.C.C + */ +public class TermFrequency extends Counter { + + private static final long serialVersionUID = 1L; + private CharFilter filter; + + /** + * Default constructor + */ + public TermFrequency() { + filter = null; + } + + /** + * Basic constructor + * + * @param filter a CharFilter implementing character equivalences + */ + public TermFrequency(CharFilter filter) { + this.filter = filter; + } + + /** + * Add CharFilter + * + * @param file a CSV file with character equivalences + */ + public void addFilter(File file) { + if (filter == null) { + filter = new CharFilter(file); + } else { + filter.addFilter(file); + } + } + + /** + * Extract words from a file + * + * @param dir the input file or directory + * @throws eu.digitisation.input.WarningException + * @throws eu.digitisation.input.SchemaLocationException + */ + public void add(File dir) throws WarningException, SchemaLocationException { + if (dir.isDirectory()) { + addFiles(dir.listFiles()); + } else { + File[] files = {dir}; + addFiles(files); + } + } + + /** + * Extract words from a file + * + * @param file an input files + * @throws eu.digitisation.input.WarningException + * @throws eu.digitisation.input.SchemaLocationException + */ + public void addFile(File file) throws WarningException, SchemaLocationException { + try { + Text content = new Text(file); + WordScanner scanner = new WordScanner(content.toString(), + "[^\\p{Space}]+"); + String word; + while ((word = scanner.nextWord()) != null) { + String filtered = (filter == null) + ? word : filter.translate(word); + inc(StringNormalizer.composed(filtered)); + } + } catch (IOException ex) { + Messages.info(TermFrequency.class.getName() + ": " + ex); + } + + } + + /** + * Extract words from a file + * + * @param files an array of input files + */ + private void addFiles(File[] files) throws WarningException, SchemaLocationException { + for (File file : files) { + addFile(file); + } + } + + /** + * + * @param other another term frequency vector + * @return the cosine distance (normalized scalar product) + */ + public double cosine(TermFrequency other) { + double norm1 = 0; + double norm2 = 0; + double scalar = 0; + + for (Map.Entry entry : this.entrySet()) { + int freq = entry.getValue(); + norm1 += freq * freq; + scalar += freq * other.get(entry.getKey()); + } + + for (int freq2 : other.values()) { + norm2 += freq2 * freq2; + } + + return scalar / Math.sqrt(norm1 * norm2); + } + + /** + * + * @param other another term frequency vector + * @return the recall provided by this term frequency vector (rate of words + * in the other TF matching one in this TF) + */ + public double recall(TermFrequency other) { + int total = 0; + int matched = 0; + + for (Map.Entry entry : other.entrySet()) { + total += entry.getValue(); + if (this.containsKey(entry.getKey())) { + ++matched; + } + } + + return matched / (double) total; + } + + /** + * Compute the order-independent edit-distance between two documents + * (equivalent to a bags of words model). + * + * @param other another TermFrequency + * @return the number of differences between this and the other bag of words + */ + public int editDistance(TermFrequency other) { + int dplus = 0; // excess + int dminus = 0; // fault + for (String word : this.keySet()) { + int delta = this.value(word) - other.value(word); + if (delta > 0) { + dplus += delta; + } else { + dminus += delta; + } + } + for (String word : other.keySet()) { + if (!this.containsKey(word)) { + int delta = this.value(word) - other.value(word); + if (delta > 0) { + dplus += delta; + } else { + dminus += delta; + } + } + } + return Math.max(dplus, dminus); + } + + /** + * String representation + * + * @param order the criteria to sort words + * @return String representation + */ + public String toString(Order order) { + StringBuilder builder = new StringBuilder(); + for (String word : this.keyList(order)) { + builder.append(word).append(' ') + .append(get(word)).append('\n'); + } + return builder.toString(); + } + + /** + * Main function + * + * @param args see help + * @throws java.lang.Exception + */ + public static void main(String[] args) throws Exception { + if (args.length < 1) { + System.err.println("Usage: TermFrequency [-e equivalences_file] [-c] input_files_or_directories"); + } else { + TermFrequency tf = new TermFrequency(); + List files = new ArrayList(); + CharFilter filter = new CharFilter(); + for (int n = 0; n < args.length; ++n) { + if (args[n].equals("-e")) { + tf.addFilter(new File(args[++n])); + } else if (args[n].equals("-c")) { + filter.setCompatibility(true); + } else { + files.add(new File(args[n])); + } + } + for (File file : files) { + tf.add(file); + } + System.out.println(tf.toString(Order.DESCENDING_VALUE)); + } + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/layout/ALTOPage.java b/ocrevalUAtion/src/main/java/eu/digitisation/layout/ALTOPage.java new file mode 100644 index 00000000..4bd6505d --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/layout/ALTOPage.java @@ -0,0 +1,125 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.layout; + +import eu.digitisation.input.FileType; +import eu.digitisation.log.Messages; +import eu.digitisation.xml.DocumentParser; +import java.awt.Polygon; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +/** + * + * @author R.C.C. + */ +public class ALTOPage extends Page { + + public ALTOPage(File file) { + try { + parse(file); + } catch (IOException ex) { + Messages.info(ALTOPage.class.getName() + ": " + ex); + } + } + + /** + * Read the bounding-box information stored in the element attributes l, t, + * b, r + * + * @param e the container element + * @return the BoundingBox for this element or null if some attributes are + * missing + */ + private static BoundingBox getBBox(Element e) { + if (e.hasAttribute("HEIGHT") + && e.hasAttribute("WIDTH") + && e.hasAttribute("VPOS") + && e.hasAttribute("HPOS")) { + int height = Integer.parseInt(e.getAttribute("HEIGHT")); + int width = Integer.parseInt(e.getAttribute("WIDTH")); + int y0 = Integer.parseInt(e.getAttribute("VPOS")); + int x0 = Integer.parseInt(e.getAttribute("HPOS")); + return new BoundingBox(x0, y0, x0 + width, y0 + height); + } else { + return null; + } + } + + @Override + public final void parse(File file) throws IOException { + Document doc = DocumentParser.parse(file); + String id = ""; + ComponentType type = ComponentType.PAGE; + + NodeList nodes = doc.getElementsByTagName("Page"); + if (nodes.getLength() > 1) { + throw new IOException("Multiple pages in document"); + } else { + Element page = (Element) nodes.item(0); + root = parse(page); + } + + } + + /** + * Parse an XML element and retrieve the associated text component + * + * @param element an XML element + * @return the text component associated with this element + */ + TextComponent parse(Element element) { + String id = element.getAttribute("ID"); + ComponentType type = ComponentType.valueOf(FileType.ALTO, element.getTagName()); + String subtype = element.getAttribute("TYPE"); + String content = element.getAttribute("CONTENT"); + BoundingBox bbox = getBBox(element); + Polygon frontier = (bbox == null) ? null : bbox.asPolygon(); + List array = new ArrayList(); + NodeList children = element.getChildNodes(); + int number = components.size(); + + components.add(null); // reserve the place for this component + + for (int nchild = 0; nchild < children.getLength(); ++nchild) { + Node child = children.item(nchild); + if (child instanceof Element) { + Element e = (Element) child; + String tag = e.getTagName(); + if (tag.equals("PrintSpace") + || tag.equals("ComposedBlock") + || tag.equals("TextBlock") + || tag.equals("TextLine") + || tag.equals("String")) { + TextComponent subcomponent = parse(e); + array.add(subcomponent); + } + } + } + TextComponent component = new TextComponent(id, type, subtype, content, frontier); + components.set(number, component); + subcomponents.put(component, array); + return component; + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/layout/BoundingBox.java b/ocrevalUAtion/src/main/java/eu/digitisation/layout/BoundingBox.java new file mode 100644 index 00000000..bf64bb13 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/layout/BoundingBox.java @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.layout; + +import java.awt.Polygon; +import java.awt.Rectangle; + +/** + * Bounding box = coordinates of the rectangular border that fully encloses the + * digital image + * + * @author R.C.C. + */ +public class BoundingBox extends Rectangle { + + private static final long serialVersionUID = 1L; + + /** + * Creates an empty BoundingBox, that is a rectangle with coordinates + * (x0, y0, x1, y1) = (0, 0, 0, 0) + */ + public BoundingBox() { + super(); + } + + /** + * Create a bounding box with the specified corners + * + * @param x0 upper left corner x-coordinate + * @param y0 upper-left corner y-coordinate + * @param x1 lower-right corner x-coordinate + * @param y1 lower-right corner y-coordinate + */ + public BoundingBox(int x0, int y0, int x1, int y1) { + super(x0, y0, x1 - x0, y1 - y0); + } + + /** + * Build a bounding box for a polygon + * + * @param polygon + */ + public BoundingBox(Polygon polygon) { + super(polygon.getBounds()); + } + + /** + * The bounding box a s a polygon + * + * @return + */ + public Polygon asPolygon() { + Polygon polygon = new Polygon(); + polygon.addPoint(x, y); + polygon.addPoint(x, y + height); + polygon.addPoint(x + width, y + height); + polygon.addPoint(x + width, y); + return polygon; + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/layout/ComponentTag.java b/ocrevalUAtion/src/main/java/eu/digitisation/layout/ComponentTag.java new file mode 100644 index 00000000..fa80fb42 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/layout/ComponentTag.java @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.layout; + +import eu.digitisation.input.FileType; + +/** + * The tag of a text component in a document + * + * @author R.C.C. Info about tags FR10 + * http://www.abbyy.com/FineReader_xml/FineReader10-schema-v1.xml ALTO + * http://www.loc.gov/standards/alto/techcenter/layout.php hOCR + * http://docs.google.com/View?docid=dfxcv4vc_67g844kf + */ +public enum ComponentTag { + + PAGE_Page, PAGE_TextRegion, PAGE_TextLine, PAGE_Word, + HOCR_ocr_page, HOCR_ocr_carea, HOCR_ocr_par, + HOCR_ocr_line, HOCR_ocr_word, HOCR_ocrx_word, + FR10_page, FR10_block, FR10_text, FR10_par, FR10_line, + FR10_formatting, FR10_word, + ALTO_Page, ALTO_PrintSpace, ALTO_ComposedBlock, ALTO_TextBlock, + ALTO_TextLine, ALTO_String; + + /** + * The type for a given tag and type of file + * + * @param ftype the type of file (PAGE, hOCR, ALTO, etc) + * @param tag the component tag (TextLine, ocr_par, etc) + * @return the component type associated to this tag and type of file + */ + public static ComponentTag valueOf(FileType ftype, String tag) { + return valueOf(ftype.toString() + "_" + tag); + } + + /** + * Return the component tag without the file-type prefix + * + * @param tag a ComponentTag + * @return the tag for this component without the file-type prefix + */ + public static String shortTag(ComponentTag tag) { + return tag.toString().replaceFirst("[^_]+_", ""); + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/layout/ComponentType.java b/ocrevalUAtion/src/main/java/eu/digitisation/layout/ComponentType.java new file mode 100644 index 00000000..3bc97223 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/layout/ComponentType.java @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.layout; + +import eu.digitisation.input.FileType; +import java.util.EnumMap; + +/** + * Types of text components in a document + * + * @author R.C.C. + */ +public enum ComponentType { + + PAGE, BLOCK, LINE, WORD, OTHER; + + final static EnumMap types + = new EnumMap(ComponentTag.class); + + static { + types.put(ComponentTag.PAGE_Page, PAGE); + types.put(ComponentTag.PAGE_TextRegion, BLOCK); + types.put(ComponentTag.PAGE_TextLine, LINE); + types.put(ComponentTag.PAGE_Word, WORD); + types.put(ComponentTag.HOCR_ocr_page, PAGE); + types.put(ComponentTag.HOCR_ocr_carea, OTHER); // page content-area + types.put(ComponentTag.HOCR_ocr_par, BLOCK); + types.put(ComponentTag.HOCR_ocr_line, LINE); + types.put(ComponentTag.HOCR_ocr_word, WORD); + types.put(ComponentTag.HOCR_ocrx_word, WORD); + types.put(ComponentTag.FR10_page, PAGE); + types.put(ComponentTag.FR10_block, BLOCK); + types.put(ComponentTag.FR10_text, OTHER); // text in block + types.put(ComponentTag.FR10_par, LINE); + types.put(ComponentTag.FR10_line, LINE); + types.put(ComponentTag.FR10_formatting, OTHER); // text in line + types.put(ComponentTag.FR10_word, WORD); + types.put(ComponentTag.ALTO_Page, PAGE); + types.put(ComponentTag.ALTO_PrintSpace, OTHER); // page main area + types.put(ComponentTag.ALTO_ComposedBlock, OTHER); + types.put(ComponentTag.ALTO_TextBlock, BLOCK); + types.put(ComponentTag.ALTO_TextLine, LINE); + types.put(ComponentTag.ALTO_String, WORD); + } + + public static ComponentType valueOf(ComponentTag tag) { + return types.get(tag); + } + + /** + * + * @param ftype the type of files + * @param tag the tag of the component + * @return the component type for this tag and type of file + */ + public static ComponentType valueOf(FileType ftype, String tag) { + return types.get(ComponentTag.valueOf(ftype, tag)); + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/layout/FR10Page.java b/ocrevalUAtion/src/main/java/eu/digitisation/layout/FR10Page.java new file mode 100644 index 00000000..3d2b6f41 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/layout/FR10Page.java @@ -0,0 +1,237 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.layout; + +import eu.digitisation.input.FileType; +import eu.digitisation.log.Messages; +import eu.digitisation.xml.DocumentParser; +import java.awt.Polygon; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +/** + * FR10Page information contained in one GT or OCR file. Pending: store nested + * structure + * + * @author R.C.C. + */ +public class FR10Page extends Page { + + public FR10Page(File file) { + try { + parse(file); + } catch (IOException ex) { + Messages.info(FR10Page.class.getName() + ": " + ex); + } + + } + + /** + * Read the bounding-box information stored in the element attributes l, t, + * b, r + * + * @param e the container element + * @return the BoundingBox for this element or null if some attributes are + * missing + */ + private static BoundingBox getBBox(Element e) { + if (e.hasAttribute("l") + && e.hasAttribute("t") + && e.hasAttribute("r") + && e.hasAttribute("b")) { + int x0 = Integer.parseInt(e.getAttribute("l")); + int y0 = Integer.parseInt(e.getAttribute("t")); + int x1 = Integer.parseInt(e.getAttribute("r")); + int y1 = Integer.parseInt(e.getAttribute("b")); + return new BoundingBox(x0, y0, x1, y1); + } else { + return null; + } + } + + /** + * Get the textual content under a given element + * + * @param e the container element + * @return the text contained under this element + */ + private static String getTextContent(Element e) { + StringBuilder builder = new StringBuilder(); + NodeList nodes = e.getElementsByTagName("charParams"); + if (nodes.getLength() > 0) { + int last = 0; // recognise new line + for (int nchar = 0; nchar < nodes.getLength(); ++nchar) { + Element charParam = (Element) nodes.item(nchar); + String content = charParam.getTextContent(); + int left = Integer.parseInt(charParam.getAttribute("l")); + if (left < last) { + builder.append('\n'); + } + last = left; + builder.append(content.length() > 0 ? content : ' '); + } + } else if (!e.getNodeName().equals("formatting")) { + nodes = e.getElementsByTagName("formatting"); + for (int nline = 0; nline < nodes.getLength(); ++nline) { + Element charParam = (Element) nodes.item(nline); + String content = charParam.getTextContent(); + if (builder.length() > 0) { + builder.append('\n'); + } + builder.append(content); + } + } else { // a plain formatting element + builder.append(e.getTextContent()); + } + + return builder.toString(); + } + + @Override + public final void parse(File file) throws IOException { + + Document doc = DocumentParser.parse(file); + String id = ""; + ComponentType type = ComponentType.PAGE; + NodeList nodes = doc.getElementsByTagName("page"); + if (nodes.getLength() > 1) { + throw new IOException("Multiple pages in document"); + } else { + Element page = (Element) nodes.item(0); + root = parse(page); + } + } + + private TextComponent parse(Element element) { + String id = element.getAttribute("pageElemId"); + ComponentType type = ComponentType.valueOf(FileType.FR10, element.getTagName()); + String subtype = element.getAttribute("blockType"); // empty for non-blocks + String content = getTextContent(element); + BoundingBox bbox = getBBox(element); + Polygon frontier = (bbox == null) ? null : bbox.asPolygon(); + + List array = new ArrayList(); + NodeList children = element.getChildNodes(); + int number = components.size(); + + components.add(null); // add room for this component + + for (int nchild = 0; nchild < children.getLength(); ++nchild) { + Node child = children.item(nchild); + if (child instanceof Element) { + Element subelement = (Element) child; + String tag = subelement.getTagName(); + + if (tag.equals("block") + || tag.equals("text") + || tag.equals("par") + || tag.equals("line")) { + TextComponent subcomponent = parse(subelement); + array.add(subcomponent); + } else if (tag.equals("formatting")) { + List words = parseWords(subelement); + array.addAll(words); + } + } + } + + TextComponent component = new TextComponent(id, type, subtype, + content, frontier); + components.set(number, component); + subcomponents.put(component, array); + + return component; + } + + /** + * Polygonal cover of a sequence of bounding boxes + * @param points + * @return + */ + private Polygon cover(Polygon points) { + Polygon cover = new Polygon(); + for (int n = 0; n < points.npoints; ++n) { + if (n % 2 == 0) { + cover.addPoint(points.xpoints[n], points.ypoints[n]); + } else { + cover.addPoint(points.xpoints[n], points.ypoints[n - 1]); + } + } + for (int n = points.npoints - 1; n >= 0; --n) { + if (n % 2 == 1) { + cover.addPoint(points.xpoints[n], points.ypoints[n]); + } else { + cover.addPoint(points.xpoints[n], points.ypoints[n + 1]); + } + } + return cover; + } + + /** + * Specific function to split FR10 formatting elements into smaller + * components (words). Formatting is a sequence of charParams elements + * containing characters. An empty charParams element indicates word + * boundaries. + * + * @param element a formatting element + * @return the TextComponents (words) in this element + */ + private List parseWords(Element element) { + List words = new ArrayList(); + NodeList charParams = element.getElementsByTagName("charParams"); + StringBuilder builder = new StringBuilder(); + Polygon points = new Polygon(); + + for (int nchar = 0; nchar < charParams.getLength(); ++nchar) { + Element charParam = (Element) charParams.item(nchar); + String content = charParam.getTextContent().trim(); + if (!content.matches("\\p{Space}*")) { // end of word + int x0 = Integer.parseInt(charParam.getAttribute("l")); + int y0 = Integer.parseInt(charParam.getAttribute("t")); + int x1 = Integer.parseInt(charParam.getAttribute("r")); + int y1 = Integer.parseInt(charParam.getAttribute("b")); + points.addPoint(x0, y0); + points.addPoint(x1, y1); + builder.append(content); + } else if (builder.length() > 0) { // some content + TextComponent word + = new TextComponent(null, ComponentType.WORD, null, + builder.toString(), cover(points)); + words.add(word); + builder = new StringBuilder(); + points = new Polygon(); + } + } + // flush + if (builder.length() > 0) { + TextComponent word + = new TextComponent(null, ComponentType.WORD, null, + builder.toString(), cover(points)); + words.add(word); + } + components.addAll(words); + return words; + } + +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/layout/HOCRPage.java b/ocrevalUAtion/src/main/java/eu/digitisation/layout/HOCRPage.java new file mode 100644 index 00000000..8b782315 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/layout/HOCRPage.java @@ -0,0 +1,126 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.layout; + +import eu.digitisation.input.FileType; +import eu.digitisation.log.Messages; +import java.awt.Polygon; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + + +/** + * + * @author R.C.C. + */ +public class HOCRPage extends Page { + + /** + * Basic constructor + * + * @param file + */ + public HOCRPage(File file) { + try { + parse(file); + } catch (IOException ex) { + Messages.info(HOCRPage.class.getName() + ": " + ex); + } + } + + @Override + public final void parse(File file) throws IOException { + org.jsoup.nodes.Document doc = org.jsoup.Jsoup.parse(file, null); + + String id = ""; + ComponentType type = ComponentType.PAGE; + org.jsoup.select.Elements pages = doc.body().select("*[class=ocr_page"); + + if (pages.size() > 1) { + throw new IOException("Multiple pages in document"); + } else { + org.jsoup.nodes.Element page = pages.first(); + root = parse(page); + } + + } + + /** + * Parse an XML element and retrieve the associated text component + * + * @param element an HTML element + * @return the text component associated with this element + */ + TextComponent parse(org.jsoup.nodes.Element element) { + String id = element.attr("id"); + String subtype = element.attr("class"); + ComponentType type = ComponentType.valueOf(FileType.HOCR, subtype); + String content = element.text(); + Polygon frontier = null; + List array = new ArrayList(); + + // extract coordinates + String title = element.attr("title").trim(); + if (title.contains("bbox")) { + int pos = title.indexOf("bbox"); + String[] coords = title.substring(pos).split("\\p{Space}+"); + int x0 = Integer.parseInt(coords[1]); + int y0 = Integer.parseInt(coords[2]); + int x1 = Integer.parseInt(coords[3]); + int y1 = Integer.parseInt(coords[4]); + + frontier = new BoundingBox(x0, y0, x1, y1).asPolygon(); + } else if (title.contains("poly")) { + int pos = title.indexOf("poly"); + String[] coords = title.substring(pos).split("\\p{Space}+"); + int n = 1; + frontier = new Polygon(); + while (n + 1 < coords.length && !coords[n].equals(";")) { + int x = Integer.parseInt(coords[n]); + int y = Integer.parseInt(coords[n + 1]); + frontier.addPoint(x, y); + n += 2; + } + } + // get subcomponents + + org.jsoup.select.Elements children = element.children(); + + for (org.jsoup.nodes.Element child : children) { + if (child.hasAttr("class")) { + String cat = child.attr("class"); + if (cat.equals("ocr_carea") + || cat.equals("ocr_par") + || cat.equals("ocr_line") + || cat.equals("ocr_word") + || cat.equals("ocrx_word")) { + TextComponent subcomponent = parse(child); + array.add(subcomponent); + } + } + } + + TextComponent component = new TextComponent(id, type, subtype, content, frontier); + components.add(component); + System.out.println(component); + subcomponents.put(component, array); + return component; + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/layout/PAGEPage.java b/ocrevalUAtion/src/main/java/eu/digitisation/layout/PAGEPage.java new file mode 100644 index 00000000..ac168fb4 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/layout/PAGEPage.java @@ -0,0 +1,124 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.layout; + +import eu.digitisation.input.FileType; +import eu.digitisation.log.Messages; +import eu.digitisation.xml.DocumentParser; +import java.awt.Polygon; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.w3c.dom.DOMException; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +/** + * + * @author R.C.C. + */ +public class PAGEPage extends Page { + + public PAGEPage(File file) { + try { + parse(file); + } catch (IOException ex) { + Messages.info(PAGEPage.class.getName() + ": " + ex); + } + } + + @Override + public final void parse(File file) throws IOException { + Document doc = DocumentParser.parse(file); + String id = doc.getDocumentElement().getAttribute("pcGtsId"); + ComponentType type = ComponentType.PAGE; + + NodeList nodes = doc.getElementsByTagName("Page"); + if (nodes.getLength() > 1) { + throw new IOException("Multiple pages in document"); + } else { + Element page = (Element) nodes.item(0); + root = parse(page); + } + + } + + /** + * Parse an XML element and retrieve the associated text component + * + * @param element an XML element + * @return the text component associated with this element + */ + TextComponent parse(Element element) { + String id = element.getAttribute("id"); + ComponentType type = + ComponentType.valueOf(FileType.PAGE, element.getTagName()); + String subtype = element.getAttribute("type"); + String content = null; + Polygon frontier = new Polygon(); + List array = new ArrayList(); + NodeList children = element.getChildNodes(); + int number = components.size(); + + components.add(null); // reserve the place for this component + + for (int nchild = 0; nchild < children.getLength(); ++nchild) { + Node child = children.item(nchild); + if (child instanceof Element) { + Element e = (Element) child; + String tag = e.getTagName(); + if (tag.equals("TextEquiv")) { + if (content != null) { + throw new DOMException(DOMException.INVALID_ACCESS_ERR, + "Multiple content in region " + id); + } + content = child.getTextContent().trim(); + } + if (tag.equals("Coords")) { + Element coords = (Element) child; + if (frontier.npoints > 0) { + throw new DOMException(DOMException.INVALID_ACCESS_ERR, + "Multiple Coords in region " + id); + } + NodeList nodes = coords.getChildNodes(); // points + for (int npoint = 0; npoint < nodes.getLength(); ++npoint) { + Node node = nodes.item(npoint); + if (node.getNodeName().equals("Point")) { + Element point = (Element) node; + int x = Integer.parseInt(point.getAttribute("x")); + int y = Integer.parseInt(point.getAttribute("y")); + frontier.addPoint(x, y); + } + } + } else if (tag.equals("TextRegion") + || tag.equals("TextLine") + || tag.equals("Word")) { + TextComponent subcomponent = parse(e); + array.add(subcomponent); + } + } + } + TextComponent component = new TextComponent(id, type, subtype, content, frontier); + components.set(number, component); + subcomponents.put(component, array); + return component; + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/layout/Page.java b/ocrevalUAtion/src/main/java/eu/digitisation/layout/Page.java new file mode 100644 index 00000000..58d88a3b --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/layout/Page.java @@ -0,0 +1,251 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.layout; + +import eu.digitisation.input.FileType; +import eu.digitisation.input.SchemaLocationException; +import java.awt.Polygon; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Interface to process input files and extract content and geometry + * + * @author R.C.C. + */ +public abstract class Page { + + TextComponent root; // the main component + List components; // all components + Map> subcomponents; // contained components + + /** + * The basic constructor + */ + Page() { + components = new ArrayList(); + subcomponents = new HashMap>(); + } + + /** + * Parse an input file + * + * @param file the input (XML or HTML) file + */ + public abstract void parse(File file) throws IOException; + + /** + * Get all the components in this document + * + * @return all the components in this document + */ + public List getComponents() { + return components; + } + + /** + * Get all subcomponents of a given component + * + * @param component a component of the document being parsed + * @return all subcomponents of this component + */ + public List getComponents(TextComponent component) { + return subcomponents.get(component); + } + + /** + * List only components of a given type + * + * @param type a component type + * @return the list of components with this type + */ + public List getComponents(ComponentType type) { + List list = new ArrayList(); + for (TextComponent component : components) { + if (component.getType() == type) { + list.add(component); + } + } + return list; + } + + /** + * List subcomponents of a given type + * + * @param type a component type + * @return the list of components with this type + */ + public List getComponents(TextComponent component, ComponentType type) { + List list = new ArrayList(); + for (TextComponent subcomponent : subcomponents.get(component)) { + if (subcomponent.getType() == type) { + list.add(subcomponent); + } + } + return list; + } + + /** + * Get the textual content of the document + * + * @return the textual content of the document + */ + public String getText() { + return root.getContent(); + } + + /** + * Get the text content of a given component + * + * @param component a component of the document being parsed + * @return text under this component + */ + public String getText(TextComponent component) { + return component.getContent(); + } + + /** + * Get the text content of all components of a given type + * + * @param type a component type + * @return text content under components of this type + */ + public String getText(ComponentType type) { + StringBuilder builder = new StringBuilder(); + for (TextComponent component : components) { + if (component.getType() == type) { + if (builder.length() > 0) { + builder.append(' '); + } + builder.append(component.getContent()); + } + } + return builder.toString(); + } + + /** + * Text content in subcomponents of a given type + * + * @param component a component + * @param type a component type + * @return the text content under subcomponents with this type + */ + public String getText(TextComponent component, ComponentType type) { + StringBuilder builder = new StringBuilder(); + for (TextComponent subcomponent : subcomponents.get(component)) { + if (subcomponent.getType() == type) { + if (builder.length() > 0) { + builder.append(' '); + } + builder.append(subcomponent.getContent()); + } + } + return builder.toString(); + } + + /** + * Transform a list of components into a list of polygonal frontiers + * + * @param components + * @return + */ + private List frontiers(List components) { + List frontiers = new ArrayList(components.size()); + for (TextComponent component : components) { + frontiers.add(component.getFrontier()); + } + return frontiers; + } + + /** + * Get all the components in this document + * + * @return all the components in this document + */ + public List getFrontiers() { + return frontiers(components); + } + + /** + * Get the frontier a given component + * + * @param component a component of the document being parsed + * @return all subcomponents of this component + */ + public Polygon getFrontier(TextComponent component) { + return component.getFrontier(); + } + + /** + * List of (non-null) frontiers of components with a given type + * + * @param type a component type + * @return the list of polygonal frontiers of components with this type + */ + public List getFrontiers(ComponentType type) { + List list = new ArrayList(); + for (TextComponent component : components) { + if (component.getType() == type) { + Polygon p = component.getFrontier(); + if (p != null) { + list.add(p); + } + } + } + return list; + } + + /** + * List (non-null) frontiers of subcomponents with a given type + * + * @param type a component type + * @return the list of frontiers of subcomponents with this type + */ + public List getFrontiers(TextComponent component, ComponentType type) { + List list = new ArrayList(); + for (TextComponent subcomponent : subcomponents.get(component)) { + if (subcomponent.getType() == type) { + Polygon p = subcomponent.getFrontier(); + if (p != null) { + list.add(p); + } + list.add(subcomponent.getFrontier()); + } + } + return list; + } + + public static void main(String[] args) + throws SchemaLocationException, IOException { + File file = new File(args[0]); + FileType ftype = FileType.valueOf(file); + + if (ftype == FileType.PAGE) { + Page page = new PAGEPage(file); + System.out.println(""); + for (TextComponent component : page.getComponents()) { + System.out.println(component); + } + System.out.println(""); + } + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/layout/SortPageXML.java b/ocrevalUAtion/src/main/java/eu/digitisation/layout/SortPageXML.java new file mode 100644 index 00000000..26cdf712 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/layout/SortPageXML.java @@ -0,0 +1,213 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, transform to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.layout; + +import eu.digitisation.xml.DocumentBuilder; +import eu.digitisation.xml.DocumentParser; +import eu.digitisation.xml.DocumentWriter; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +/** + * PAGE-XML regions order in the document can differ form reading order. Such + * information is stored under OrdereGroup (recursive) elements. This class + * restores the appropriate order of elements in the document. + * + * @author R.C.C. + */ +public class SortPageXML { + + /** + * SortPageXML children consistently with the order defined for their id + * attribute Remark: NodeList mutates when operations on nodes take place: + * an backup childList is used to avoid such conflicts. + * + * @param node the parent node + * @param order the array of id's in ascending order + */ + private static void sort(Node node, List order) { + Map index = new HashMap(); // index of children nodes + NodeList children = node.getChildNodes(); + List childList = new ArrayList(); // External copy of children + + // Initialize index (only nodes which need reordering will be stored) + for (String id : order) { + index.put(id, null); + } + + // Create an index ot text regions which need reordering + for (int n = 0; n < children.getLength(); ++n) { + Node child = children.item(n); + childList.add(child); + if (child instanceof Element) { + String id = ((Element) child).getAttribute("id"); + if (index.containsKey(id)) { + index.put(id, child); + } + } + } + + // Clear internal list of child nodes + while (children.getLength() > 0) { + node.removeChild(children.item(0)); + } + + int norder = 0; // the posititon in the order list + for (int n = 0; n < childList.size(); ++n) { + Node child = childList.get(n); + if (child instanceof Element) { + String id = ((Element) child).getAttribute("id"); + + if (index.containsKey(id)) { + Node replacement = index.get(order.get(norder)); + node.appendChild(replacement); + ++norder; + } else { + node.appendChild(child); + } + } else { + node.appendChild(child); + } + } + } + + /** + * Extract reading order as defined by an OrderedGroup element + * + * @param e the OrderedGroup element + * @return list of identifiers in reading order + * @throws IOException + */ + private static List readingOrder(Node node) throws IOException { + NodeList children = node.getChildNodes(); + List order = new ArrayList(); + + for (int n = 0; n < children.getLength(); ++n) { + Node child = children.item(n); + if (child instanceof Element + && child.getNodeName().equals("RegionRefIndexed")) { + String index = ((Element) child).getAttribute("index"); + assert Integer.parseInt(index) == order.size(); + String idref = ((Element) child).getAttribute("regionRef"); + order.add(idref); + } + } + return order; + } + + /** + * + * @param doc a PAGE XML document + * @return true if doc is transformed according to the reading order + * @throws IOException + */ + public static boolean isSorted(Document doc) throws IOException { + NodeList groups = doc.getElementsByTagName("OrderedGroup"); + for (int n = 0; n < groups.getLength(); ++n) { + Node group = groups.item(n); + List order = readingOrder(group); + Node container = group.getParentNode().getParentNode(); + NodeList children = container.getChildNodes(); + + int nreg = 0; // region number in group + for (int k = 0; k < children.getLength(); ++k) { + Node child = children.item(k); + if (child instanceof Element) { + String tag = child.getNodeName(); + if (tag.equals("TextRegion")) { + String id = ((Element) child).getAttribute("id"); + if (order.contains(id) && // sometimes item is not listed + !id.equals(order.get(nreg++))) { + return false; + } + } + } + } + } + return true; + } + + /** + * + * @param file a PAGE XML input file + * @return true if the document in file is transformed according to the + * reading order + * @throws IOException + */ + public static boolean isSorted(File file) throws IOException { + Document doc = DocumentParser.parse(file); + return isSorted(doc); + } + + /** + * Create document where ordered groups follow the order information + * + * @param source the input document to be transformed + * @return the transformed document + * @throws java.io.IOException + */ + public static Document sorted(Document source) throws IOException { + Document doc = DocumentBuilder.clone(source); + NodeList groups = doc.getElementsByTagName("OrderedGroup"); + for (int n = 0; n < groups.getLength(); ++n) { + Node group = groups.item(n); + // group element -> ReadingOrder-> OrderedGroup + Node container = group.getParentNode().getParentNode(); + List order = readingOrder(group); + sort(container, order); + } + return doc; + } + + /** + * Create a file where ordered groups follow the order information + * + * @param ifile the input PAGE XML file + * @param ofile the file with the transformed document + * @throws java.io.IOException + */ + public static void transform(File ifile, File ofile) throws IOException { + Document doc = DocumentParser.parse(ifile); + DocumentWriter writer = new DocumentWriter(SortPageXML.sorted(doc)); + writer.write(ofile); + } + + /** + * Demo main + * + * @param args + * @throws IOException + */ + public static void main(String[] args) throws IOException { + File ifile = new File(args[0]); + + System.out.println(SortPageXML.isSorted(ifile)); + if (args.length > 1) { + File ofile = new File(args[1]); + SortPageXML.transform(ifile, ofile); + } + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/layout/TextComponent.java b/ocrevalUAtion/src/main/java/eu/digitisation/layout/TextComponent.java new file mode 100644 index 00000000..ec8b7e05 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/layout/TextComponent.java @@ -0,0 +1,121 @@ +package eu.digitisation.layout; + +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +import java.awt.Polygon; + +/** + * A region in a document (a page, a block, line, or word) + * + * @author R.C.C. + */ +public class TextComponent { + + private static final long serialVersionUID = 1L; + + String id; // identifier + ComponentType type; // the type of component (page, block, line, word) + String subtype; // the type of block (paragraph, header, TOC). + String content; // text content + Polygon frontier; // the component frontier + + /** + * Basic constructor + * + * @param id identifier + * @param type the type of component (page, block, line, word) + * @param subtype the type of block (paragraph, header, TOC). + * @param content text content + * @param frontier the component frontier + */ + public TextComponent(String id, ComponentType type, String subtype, + String content, Polygon frontier) { + this.id = id; + this.type = type; + this.subtype = subtype; + this.content = content; + this.frontier = frontier; + } + + /** + * + * @return the component identifier + */ + public String getId() { + return id; + } + + /** + * + * @return the type of this component + */ + public ComponentType getType() { + return type; + } + + /** + * + * @return the subtype of this component + */ + public String getSubtype() { + return subtype; + } + + /** + * Get the text content + * + * @return the textual contend of this component + */ + public String getContent() { + return content; + } + + /** + * Get the frontier + * + * @return the polygonal frontier of this component + */ + public Polygon getFrontier() { + return frontier; + } + + /** + * + * @return a string representation of this TextComponent + */ + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + + builder.append("\n") + .append("\t").append(id).append("\n") + .append("\t").append(type).append("\n") + .append("\t").append(subtype).append("\n") + .append("\t").append(content).append("\n") + .append("\t\n"); + if (frontier != null) { + for (int n = 0; n < frontier.npoints; ++n) { + builder.append("\t\t").append(frontier.xpoints[n]).append("") + .append("").append(frontier.ypoints[n]).append("\n"); + } + } + builder.append("\t\n"); + builder.append("\n"); + return builder.toString(); + } + } diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/layout/Viewer.java b/ocrevalUAtion/src/main/java/eu/digitisation/layout/Viewer.java new file mode 100644 index 00000000..45aaea3d --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/layout/Viewer.java @@ -0,0 +1,138 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.layout; + +import eu.digitisation.image.Bimage; +import eu.digitisation.input.FileType; +import eu.digitisation.input.SchemaLocationException; +import java.awt.Color; +import java.awt.Desktop; +import java.io.File; +import java.io.IOException; + +/** + * Shows text regions (as stored in PAGE XML) on image + * + * @author R.C.C + */ +public class Viewer { + + /** + * Split a file name into path, base-name and extension + * + * @param filename + * @return path (before last separator), base-name (before last dot) and + * extension (after last dot) + */ + private static String[] getFilenameTokens(String filename) { + String[] tokens = new String[3]; + int first = filename.lastIndexOf(File.separator); + int last = filename.lastIndexOf('.'); + tokens[0] = filename.substring(0, first); + tokens[1] = filename.substring(first + 1, last); + tokens[2] = filename.substring(last + 1); + return tokens; + } + + /** + * Demo main + * + * @param args + * @throws IOException + */ + public static void main(String[] args) throws IOException, SchemaLocationException { + if (args.length < 2) { + System.err.println("Usage: Viewer image_file page_file [options]"); + System.exit(0); + } + + File ifile = new File(args[0]); + File xmlfile = new File(args[1]); + String opts = (args.length > 2) ? args[2] : null; + FileType ftype = FileType.valueOf(xmlfile); + String[] tokens = getFilenameTokens(args[0]); + String path = tokens[0]; + String id = tokens[1]; + String ext = tokens[2]; + File ofile = new File(path + File.separator + id + "_marked." + ext); + + Bimage page = null; + Bimage scaled; + float[] shortDash = {4f, 2f}; + float[] longDash = {8f, 4f}; + + Page gt = null; + + if (ifile.exists()) { + try { + page = new Bimage(ifile).toRGB(); + } catch (NullPointerException ex) { + throw new IOException("Unsupported format"); + } + } else { + throw new java.io.IOException(ifile.getCanonicalPath() + " not found"); + } + if (xmlfile.exists()) { + switch (ftype) { + case PAGE: + gt = new PAGEPage(xmlfile); + break; + case HOCR: + gt = new HOCRPage(xmlfile); + break; + case FR10: + gt = new FR10Page(xmlfile); + break; + case ALTO: + gt = new ALTOPage(xmlfile); + break; + default: + throw new java.lang.UnsupportedOperationException("Still not implemented"); + } + } else { + throw new java.io.IOException(xmlfile.getCanonicalPath() + " not found"); + } + + if (opts == null || opts.contains("b")) { + page.add(gt.getFrontiers(ComponentType.BLOCK), Color.RED, 8f); + } + if (opts == null || opts.contains("l")) { + page.add(gt.getFrontiers(ComponentType.LINE), Color.GREEN, 2f, longDash); + } + if (opts == null || opts.contains("w")) { + page.add(gt.getFrontiers(ComponentType.WORD), Color.BLUE, 2f); + } + + + for (TextComponent component : gt.getComponents(ComponentType.WORD)) { + System.out.println(component); + // page.add(component.getFrontier(), Color.BLUE, 2f); + } + + scaled = new Bimage(page, 1.0); + scaled.write(ofile); + System.out.println("output=" + ofile); + + if (opts != null && opts.contains("s")) { + if (Desktop.getDesktop().isSupported(Desktop.Action.OPEN)) { + Desktop.getDesktop().open(ofile); + } + } + + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/log/LogFormatter.java b/ocrevalUAtion/src/main/java/eu/digitisation/log/LogFormatter.java new file mode 100644 index 00000000..d8a663f5 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/log/LogFormatter.java @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.log; + +import java.util.logging.Formatter; +import java.util.logging.LogRecord; + +/** + * + * @author R.C.C. + */ +public class LogFormatter extends Formatter { + + public LogFormatter() { + super(); + } + + @Override + public String format(LogRecord record) { + StringBuilder builder = new StringBuilder(); + + builder.append(record.getLevel().getName()); + builder.append(" "); + builder.append(formatMessage(record)); + builder.append("\n"); + + return builder.toString(); + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/log/LogFrame.java b/ocrevalUAtion/src/main/java/eu/digitisation/log/LogFrame.java new file mode 100644 index 00000000..19e418e7 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/log/LogFrame.java @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.log; + +import java.awt.Container; +import javax.swing.JFrame; +import javax.swing.JScrollPane; +import javax.swing.JTextArea; + +/** + * + * @author R.C.C. + */ +public class LogFrame extends JFrame { + + private static final long serialVersionUID = 1L; + private final Container pane; + private final JTextArea text; + + public LogFrame() { + this.setTitle("Operations log"); + setSize(600, 300); + setDefaultCloseOperation(JFrame.DISPOSE_ON_CLOSE); + text = new JTextArea(); + pane = getContentPane(); + pane.add(new JScrollPane(text)); + setVisible(true); + + } + + public void showInfo(String data) { + text.append(data); + //this.validate(); + //this.repaint(); + } + + public void close() { + setVisible(false); + dispose(); + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/log/LogFrameHandler.java b/ocrevalUAtion/src/main/java/eu/digitisation/log/LogFrameHandler.java new file mode 100644 index 00000000..8633bf64 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/log/LogFrameHandler.java @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.log; + +import java.util.logging.Handler; +import java.util.logging.LogRecord; + +/** + * + * @author R.C.C. + */ +public class LogFrameHandler extends Handler { + + private LogFrame frame = new LogFrame(); + + @Override + public void publish(LogRecord record) { + if (frame == null) { + frame = new LogFrame(); + } + if (!frame.isVisible()) { + frame.setVisible(true); + } + if (isLoggable(record)) { + String message = getFormatter().format(record); + frame.showInfo(message); + } + } + + @Override + public void flush() { + frame.revalidate(); + frame.repaint(); + } + + @Override + public void close() { + frame = null; + } + +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/log/Messages.java b/ocrevalUAtion/src/main/java/eu/digitisation/log/Messages.java new file mode 100644 index 00000000..32e59516 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/log/Messages.java @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.log; + +import java.io.File; + +/** + * + * @author R.C.C. + */ +public class Messages { + + // private static final Logger logger = Logger.getLogger("ApplicationLog"); + + static { + // try { + // URI uri = Messages.class.getProtectionDomain() + // .getCodeSource().getLocation().toURI(); + // String dir = new File(uri.getPath()).getParent(); + // File file = new File(dir, "ocrevaluation.log"); + // + // addFile(file); + // Messages.info("Logfile is " + file); + // + // // while debugging + // if (java.awt.Desktop.isDesktopSupported()) { + // addFrame(); + // } + // } catch (SecurityException ex) { + // Messages.info(Messages.class.getName() + ": " + ex); + // } catch (URISyntaxException ex) { + // Logger.getLogger(Messages.class.getName()).log(Level.SEVERE, null, + // ex); + // } + } + + public static void addFile(File file) { + // try { + // FileHandler fh = new FileHandler(file.getAbsolutePath()); + // fh.setFormatter(new LogFormatter()); + // logger.addHandler(fh); + // } catch (IOException ex) { + // Messages.info(Messages.class.getName() + ": " + ex); + // } + } + + public static void addFrame() { + // Only while debugging + // LogFrameHandler lfh = new LogFrameHandler(); + // lfh.setFormatter(new LogFormatter()); + // logger.addHandler(lfh); + } + + public static void info(String s) { + // logger.info(s); + } + + public static void warning(String s) { + // logger.warning(s); + } + + public static void severe(String s) { + // logger.severe(s); + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/math/Arrays.java b/ocrevalUAtion/src/main/java/eu/digitisation/math/Arrays.java new file mode 100644 index 00000000..6f552a2a --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/math/Arrays.java @@ -0,0 +1,280 @@ +/* + * Copyright (C) 2013 R.C.C. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.math; + +/** + * Standard operations on arrays: sum, average, max, min, standard deviation. + * + * @author R.C.C. + * @version 20131110 + */ +public class Arrays { + + /** + * @param array array of int + * @return the sum of all ints in array + */ + public static int sum(int[] array) { + int sum = 0; + + for (int n = 0; n < array.length; ++n) { + sum += array[n]; + } + return sum; + } + + /** + * + * @param array array of double + * @return the sum of all doubles in array + */ + public static double sum(double[] array) { + double sum = 0; + for (int n = 0; n < array.length; ++n) { + sum += array[n]; + } + return sum; + } + + /** + * Transform an int array into a an array of doubles + * + * @param array integer array + * @return the array of double precision values + */ + public static double[] toDouble(int[] array) { + double[] darray = new double[array.length]; + for (int i = 0; i < array.length; i++) { + darray[i] = array[i]; + } + return darray; + } + + /** + * Create an array containing the logarithms of the source array + * + * @param array integer array + * @return the array of logs + */ + public static double[] log(int[] array) { + double[] darray = new double[array.length]; + for (int i = 0; i < array.length; i++) { + darray[i] = Math.log(array[i]); + } + return darray; + } + + /** + * Create an array containing the logarithms of the source array + * + * @param array array of double + * @return the array of logs + */ + public static double[] log(double[] array) { + double[] darray = new double[array.length]; + for (int i = 0; i < array.length; i++) { + darray[i] = Math.log(array[i]); + } + return darray; + } + + /** + * @param array + * @return the average of all integers in an array + */ + public static double average(int[] array) { + return sum(array) / (double) array.length; + } + + /** + * The average of all doubles in an array + * + * @param array + * @return the average of all doubles in an array + */ + public static double average(double[] array) { + return sum(array) / array.length; + } + + /** + * @param array + * @return the geometric mean (log-average) of all integers in an array + */ + public static double logaverage(int[] array) { + return Math.exp(average(log(array))); + } + + /** + * The geometric mean (log-average) of all doubles in an array + * + * @param array + * @return the average of all doubles in an array + */ + public static double logaverage(double[] array) { + return Math.exp(average(log(array))); + } + + /** + * The scalar product + * + * @param x the first array + * @param y the second array + * @return the scalar product of x and y + */ + public static double scalar(double[] x, double[] y) { + double sum = 0; + for (int n = 0; n < x.length; ++n) { + sum += x[n] * y[n]; + } + return sum; + } + + /** + * @param array int array + * @return the max value in int array + */ + public static int max(int[] array) { + int mu = array[0]; + + for (int n = 1; n < array.length; ++n) { + mu = Math.max(mu, array[n]); + } + return mu; + } + + /** + * @param array array of doubles + * @return the max value in this array + */ + public static double max(double[] array) { + double mu = array[0]; + + for (int n = 1; n < array.length; ++n) { + mu = Math.max(mu, array[n]); + } + return mu; + } + + /** + * @param array int array + * @return the min value in int array + */ + public static int min(int[] array) { + int mu = array[0]; + + for (int n = 1; n < array.length; ++n) { + mu = Math.min(mu, array[n]); + } + return mu; + } + + /** + * @param array array of doubles + * @return the min value in this array + */ + public static double min(double[] array) { + double mu = array[0]; + + for (int n = 1; n < array.length; ++n) { + mu = Math.min(mu, array[n]); + } + return mu; + } + + /** + * @param array int array + * @return first position containing the max value in int array + */ + public static int argmax(int[] array) { + int pos = 0; + + for (int n = 1; n < array.length; ++n) { + if (array[n] > array[pos]) { + pos = n; + } + } + return pos; + } + + /** + * @param array int array + * @return first position containing the min value in int array + */ + public static int argmin(int[] array) { + int pos = 0; + + for (int n = 1; n < array.length; ++n) { + if (array[n] < array[pos]) { + pos = n; + } + } + return pos; + } + + /** + * @param X array of int + * @param Y another array of int + * @return the covariance of two variables X and Y are expected to have same + * length + */ + public static double cov(int[] X, int[] Y) { + int len = Math.min(X.length, Y.length); + double sum = 0; // double safer against overflows + + for (int n = 0; n < len; ++n) { + sum += X[n] * (double) Y[n]; + } + + return sum / len - average(X) * average(Y); + } + + /** + * Covariance of two variables + * + * @param X array of double + * @param Y another array of double + * @return Covariance of X and Y + */ + public static double cov(double[] X, double[] Y) { + int len = Math.min(X.length, Y.length); + double sum = 0; + + for (int n = 0; n < len; ++n) { + sum += X[n] * Y[n]; + } + return sum / len - average(X) * average(Y); + } + + /** + * @param X the array with the values of the variable + * @return the standard deviation of the values in X + */ + public static double std(int[] X) { + return Math.sqrt(cov(X, X)); + } + + /** + * Standard deviation + * + * @param X the array with the values of the variable + * @return the standard deviation of the values in X + */ + public static double std(double[] X) { + return Math.sqrt(cov(X, X)); + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/math/BiCounter.java b/ocrevalUAtion/src/main/java/eu/digitisation/math/BiCounter.java new file mode 100644 index 00000000..cdd2be2f --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/math/BiCounter.java @@ -0,0 +1,176 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.math; + +import java.util.Map; +import java.util.Set; + +/** + * Keeps a counter for pairs of objects (joint frequencies) and marginal counts + * + * @author R.C.C. + * @param the type of first object + * @param the type of the second object + */ +public class BiCounter, T2 extends Comparable> + extends Counter> { + + private static final long serialVersionUID = 1L; + Counter subtotal1; + Counter subtotal2; + + /** + * Default constructor + */ + public BiCounter() { + super(); + subtotal1 = new Counter(); + subtotal2 = new Counter(); + } + + /** + * + * @param o1 first component in pair to be incremented + * @param o2 second component in pair to be incremented + * @param value the value to be added for the pair count + * @return this BiCounter + */ + public BiCounter add(T1 o1, T2 o2, int value) { + Pair pair = new Pair(o1, o2); + super.add(pair, value); + subtotal1.add(o1, value); + subtotal2.add(o2, value); + return this; + } + + /** + * Set the value for a count and reset accordingly the total and marginal + * counts + * + * @param o1 first component in pair + * @param o2 second component in pair + * @param value the value for the pair count + * @return this BiCounter + */ + public BiCounter set(T1 o1, T2 o2, int value) { + Pair pair = new Pair(o1, o2); + super.set(pair, value); + subtotal1.set(o1, value); + subtotal2.set(o2, value); + return this; + } + + /** + * Add one to the count for a pair + * + * @param o1 first component in pair to be incremented + * @param o2 second component in pair to be incremented + * @return this BiCounter + */ + public BiCounter inc(T1 o1, T2 o2) { + return add(o1, o2, 1); + } + + /** + * Subtract one to the count for a pair + * + * @param o1 first component in the pair to be decremented + * @param o2 second component in the pair be decremented + * @return this BiCounter + */ + public BiCounter dec(T1 o1, T2 o2) { + return add(o1, o2, -1); + } + + /** + * Increment the count for an pair with the value stored in another + * BiCounter. + * + * @param counter the counter whose values will be added to this one. + * @return this BiCounter + */ + public BiCounter add(BiCounter counter) { + for (Map.Entry, Integer> entry : counter.entrySet()) { + Pair key = entry.getKey(); + Integer value = entry.getValue(); + add(key.first, key.second, value); + } + return this; + } + + /** + * + * @param o1 first component in pair + * @param o2 second component in pair + * @return the value of the counter for that pair, or 0 if not stored. If + * one the components is null the marginal count is returned. + * @throws NullPointerException if both objects are null + */ + public int value(T1 o1, T2 o2) { + if (o1 == null) { + return subtotal2.value(o2); + } else if (o2 == null) { + return subtotal1.value(o1); + } else { + Pair pair = new Pair(o1, o2); + return super.value(pair); + } + } + + /** + * + * @return the set of left components in pairs of the key set + */ + public Set leftKeySet() { + return subtotal1.keySet(); + } + + /** + * + * @return the marginal counts for the left ley + */ + public Counter leftSubtotal() { + return subtotal1; + } + + /** + * + * @return the set of right components in pairs of the key set + */ + public Set rightKeySet() { + return subtotal2.keySet(); + } + + /** + * + * @return the marginal counts for the right ley + */ + public Counter rightSubtotal() { + return subtotal2; + } + + /** + * Clear the BiCounter + */ + @Override + public void clear() { + super.clear(); + subtotal1.clear(); + subtotal2.clear(); + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/math/Counter.java b/ocrevalUAtion/src/main/java/eu/digitisation/math/Counter.java new file mode 100644 index 00000000..7d3d4b43 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/math/Counter.java @@ -0,0 +1,223 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.math; + +import java.text.Collator; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; + +/** + * Counts number of different objects, a map between objects and integers which + * can be incremented and decremented. + * + * @version 2012.06.07 + * @param the class of objects being counted + */ +public class Counter> + extends java.util.TreeMap { + + private static final long serialVersionUID = 1L; + + int total = 0; // stores aggregated counts + + /** + * Increment the count for an object with the given value + * + * @param object the object whose count will be incremented + * @param value the delta value + * @return this Counter + */ + public Counter add(Type object, int value) { + int storedValue; + if (containsKey(object)) { + storedValue = get(object); + } else { + storedValue = 0; + } + put(object, storedValue + value); + total += value; + return this; + } + + /** + * Set the count for an object (and the global one) with the given value + * + * @param object the object whose count will be incremented + * @param value the value for this count + * @return this Counter + */ + public Counter set(Type object, int value) { + int storedValue; + if (containsKey(object)) { + storedValue = get(object); + } else { + storedValue = 0; + } + put(object, value); + total += value - storedValue; + return this; + } + + /** + * Add one to the count for an object + * + * @param object the object whose count will be incremented + * @return this Counter + */ + public Counter inc(Type object) { + return add(object, 1); + } + + /** + * Subtract one to the count for an object + * + * @param object the object whose count will be decremented + * @return this Counter + */ + public Counter dec(Type object) { + return add(object, -1); + } + + /** + * Increment the count for an object with the value stored in another + * counter. + * + * @param counter the counter whose values will be added to this one. + * @return this Counter + */ + public Counter add(Counter counter) { + for (Map.Entry entry : counter.entrySet()) { + add(entry.getKey(), entry.getValue()); + } + return this; + } + + /** + * Increment counts for a an array of objects + * + * @param objects the array of objects + * @return this Counter + */ + public Counter add(Type[] objects) { + for (Type object : objects) { + inc(object); + } + return this; + } + + /** + * + * @param object + * @return the value of the counter for that object, or 0 if not stored + */ + public int value(Type object) { + Integer val = super.get(object); + return (val == null) ? 0 : val; + } + + /** + * Maximum value stored + * + * @return max stored value or 0 if no value is stored + */ + public int maxValue() { + if (size() > 0) { + return java.util.Collections.max(values()); + } else { + return 0; + } + } + + /** + * + * @return the aggregated count for all objects + */ + public int total() { + return total; + } + + /** + * Clear the counter + */ + @Override + public void clear() { + super.clear(); + total = 0; + } + + /** + * Specifies several orders for keys + */ + public enum Order { + + ASCENDING, DESCENDING, ASCENDING_VALUE, DESCENDING_VALUE, LEXICOGRAPHIC; + } + + private class KeyComparator implements Comparator { + + @Override + public int compare(Type first, Type second) { + int r = get(first).compareTo(get(second)); + return r; + } + } + + /** + * + * @param order determines ascending or descending order + * @return the sorted list of keys stored in the counter + */ + public List keyList(Order order) { + List list = new ArrayList(keySet()); + if (order == Order.ASCENDING) { + Collections.sort(list); + } else if (order == Order.DESCENDING) { + Collections.sort(list, Collections.reverseOrder()); + } else if (order == Order.ASCENDING_VALUE) { + Collections.sort(list, new KeyComparator()); + } else if (order == Order.DESCENDING_VALUE) { + Collections.sort(list, Collections.reverseOrder(new KeyComparator())); + } else if (order == Order.LEXICOGRAPHIC + && list.size() > 0 + && list.get(0).getClass().equals(String.class)) { + Collator collator = Collator.getInstance(); + collator.setStrength(Collator.TERTIARY); + collator.setDecomposition(Collator.FULL_DECOMPOSITION); + Collections.sort(list, collator); + } + return list; + } + + /** + * A list key-value pairs + * + * @return a string containing one key and value per line + */ + @Override + public String toString() { + StringBuilder b = new StringBuilder(); + for (Type key : keySet()) { + b.append(key).append(" ").append(get(key)).append("\n"); + } + return b.toString(); + } + +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/math/Histogram.java b/ocrevalUAtion/src/main/java/eu/digitisation/math/Histogram.java new file mode 100644 index 00000000..d1b88f6a --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/math/Histogram.java @@ -0,0 +1,152 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.math; + +import java.awt.Color; +import java.awt.Graphics2D; +import java.awt.image.BufferedImage; + +/** + * Creates toy histograms + * + * @author R.C.C. + */ +public class Histogram { + + private static final long serialVersionUID = 1L; + String title; + int[] X; + int[] Y; + + /** + * Create points for plot (one point per bar, equally spaced). + * + * @param + * @param title the title for this histogram + * @param counter a Counter + */ + public > Histogram(String title, Counter counter) { + int n = 0; + + for (Type key : counter.keySet()) { + Object obj = key; + if (obj instanceof Integer) { + Integer iobj = (Integer) obj; + X[n] = iobj.intValue(); + } else { + X[n] = n; + } + Y[n] = counter.get(key); + ++n; + } + } + + /** + * Create points for plot (one point per bar, equally spaced). + * + * @param title the title for this histogram + * @param Y an array of integer values + */ + public Histogram(String title, int[] Y) { + this.title = title; + this.Y = Y; + X = new int[Y.length]; + for (int n = 0; n < Y.length; ++n) { + X[n] = n; + } + } + + /** + * Integer exponentiation (for axis) + * + * @param base base + * @param exp exponent + * @return the exp-th power of base + */ + private int pow(int base, int exp) { + int result = 1; + while (exp != 0) { + if ((exp & 1) == 1) { + result *= base; + } + exp >>= 1; + base *= base; + } + return result; + } + + /** + * Display histogram on screen + * + * @param width display width (in pixels) + * @param height display height (in pixels) + * @param margin display margins (in pixels) + */ + public void show(int width, int height, int margin) { + BufferedImage bim + = new BufferedImage(width + 2 * margin, + height + 2 * margin, + BufferedImage.TYPE_INT_ARGB); + Graphics2D g = bim.createGraphics(); + int xhigh = Arrays.max(X); + int xlow = Arrays.min(X); + int xrange = xhigh - xlow; + int yhigh = Arrays.max(Y); + int ylow = 0; //ArrayMath.min(Y); + int yrange = yhigh - ylow; + + // draw bars + g.setColor(Color.RED); + for (int n = 0; n < X.length; ++n) { + int xpos = (width * (X[n] - xlow)) / xrange; + int ypos = (height * (Y[n] - ylow)) / yrange; + g.fillRect(margin + xpos - 1, height + margin - ypos, 3, ypos); + } + + // draw title + g.setColor(Color.DARK_GRAY); + if (title != null) { + g.drawString(title, margin, margin / 2); + } + + // draw X and Y axes + g.setColor(Color.BLUE); + g.drawRect(margin, margin, width, height); + + // draw Y-tics + int e = (int) Math.ceil(Math.log(yrange) / Math.log(10)) - 1; + int ystep = (e > 0) ? pow(10, e) : 1; + for (int y = ylow - ylow % ystep; y <= yhigh; y += ystep) { + int ypos = (height * (y - ylow)) / yrange; + g.drawString(String.valueOf(y) + "-", 0, height + margin - ypos); + } + + // draw X-tics + e = (int) Math.ceil(Math.log(xrange) / Math.log(10)) - 1; + int xstep = (e > 0) ? pow(10, e) : 1; + for (int x = xlow - xlow % xstep; x <= xhigh; x += xstep) { + int xpos = (width * (x - xlow)) / xrange; + g.drawString(String.valueOf(x), margin + xpos - 6 * e, height + margin + 12); + g.drawLine(margin + xpos, height + margin, + margin + xpos, height + margin - 5); + } + + g.dispose(); + eu.digitisation.image.Display.draw(bim); + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/math/MinimalPerfectHash.java b/ocrevalUAtion/src/main/java/eu/digitisation/math/MinimalPerfectHash.java new file mode 100644 index 00000000..f16c9efc --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/math/MinimalPerfectHash.java @@ -0,0 +1,147 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.math; + +import eu.digitisation.log.Messages; +import eu.digitisation.text.WordScanner; +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + + +/** + * Mapping between strings and integers A MinimalPerfectHash guarantees + * consistency between TokenArrays since the mapping between words and integer + * codes is shared by all TokenArrays created by the same factory and this + * allows for the comparison of TokenArrays. + * + * @version 2013.12.10 + */ +public class MinimalPerfectHash { + + /** + * The codes assigned to strings (tokens) + */ + private final HashMap codes; // token->code mapping + private final List dictionary; // code->token mapping + boolean caseSensitive; // Case sensitive encoding + + /** + * Create a new MinimalPerfectHash + * + * @param caseSensitive true if the TokenArrays must be case sensitive + */ + public MinimalPerfectHash(boolean caseSensitive) { + codes = new HashMap(); + dictionary = new ArrayList(); + this.caseSensitive = caseSensitive; + } + + /** + * Default constructor (case sensitive factory) + */ + public MinimalPerfectHash() { + this(true); + } + + /** + * + * @param word a word + * @return the integer code assigned to this word + */ + private Integer hashCode(String word) { + Integer code; + String key = caseSensitive ? word : word.toLowerCase(); + + if (codes.containsKey(key)) { + code = codes.get(key); + } else { + code = codes.size(); + codes.put(key, code); + dictionary.add(key); + } + return code; + } + + /** + * + * @param code an integer code + * @return the string or word associated with this code + */ + public String decode(int code) { + return dictionary.get(code); + } + + /** + * Build an array of hash codes from the file content + * + * @param file the input file + * @param encoding the text encoding. + * @return the list of hash codes representing the file content + */ + public List hashCodes(File file, Charset encoding) + throws RuntimeException { + ArrayList list = new ArrayList(); + + try { + WordScanner scanner = new WordScanner(file, encoding, null); + String word; + + while ((word = scanner.nextWord()) != null) { + list.add(hashCode(word)); + } + } catch (IOException ex) { + Messages.info(MinimalPerfectHash.class.getName() + ": " + ex); + } + return list; + } + + /** + * Build a TokenArray from a String + * + * @param s the input string + * @return the list of hash codes representing the file content + */ + public List hashCodes(String s) { + ArrayList list = new ArrayList(); + + try { + WordScanner scanner = new WordScanner(s, null); + String word; + + while ((word = scanner.nextWord()) != null) { + list.add(hashCode(word)); + } + } catch (IOException ex) { + Messages.info(MinimalPerfectHash.class.getName() + ": " + ex); + } + + return list; + } + + /** + * + * @return the list of all strings with a hash code in this map + */ + public List keys() { + return dictionary; + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/math/Pair.java b/ocrevalUAtion/src/main/java/eu/digitisation/math/Pair.java new file mode 100644 index 00000000..d3caa41a --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/math/Pair.java @@ -0,0 +1,89 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.math; + +/** + * A pair of objects (not necessarily of identical type) + * + * @param the type of first object + * @param the type of second object + */ +public class Pair, T2 extends Comparable> + implements Comparable> { + + public T1 first; // first element in pair + public T2 second; // second element in pair + + /** + * Default class constructor + */ + public Pair() { + } + + /** + * Class constructor + * @param first first component + * @param second second component + */ + public Pair(T1 first, T2 second) { + this.first = first; + this.second = second; + } + + /** + * Comparator + * @param other another pair + */ + @Override + public int compareTo(Pair other) { + if (this.first.equals(other.first)) { + return this.second.compareTo(other.second); + } else { + return this.first.compareTo(other.first); + } + + } + + @SuppressWarnings("unchecked") + @Override + public boolean equals(Object o) { + Pair other; + if (o == null) { + return false; + } else { + if (o instanceof Pair) { + other = (Pair) o; + } else { + throw new ClassCastException(Pair.class + + " cannot be compared with " + + o.getClass()); + } + return this.first.equals(other.first) && this.second.equals(other.second); + } + } + + @Override + public int hashCode() { + return first.hashCode() ^ 31 * second.hashCode(); + } + + @Override + public String toString() { + return "(" + first.toString() + "," + second.toString() + ")"; + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/math/Plot.java b/ocrevalUAtion/src/main/java/eu/digitisation/math/Plot.java new file mode 100644 index 00000000..9f918567 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/math/Plot.java @@ -0,0 +1,112 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.math; + +import java.awt.Color; +import java.awt.Graphics2D; +import java.awt.image.BufferedImage; + +/** + * Create a dummy plot for a function + * + * @author R.C.C. + */ +public class Plot { + + double[] X; + double[] Y; + + public Plot(double[] X, double[] Y) { + this.X = X; + this.Y = Y; + } + + /** + * Integer exponentiation (for axis) + * + * @param base base + * @param exp exponent + * @return the exp-th power of base + */ + private int pow(int base, int exp) { + int result = 1; + while (exp != 0) { + if ((exp & 1) == 1) { + result *= base; + } + exp >>= 1; + base *= base; + } + return result; + } + + /** + * Display histogram on screen + * + * @param width display width (in pixels) + * @param height display height (in pixels) + * @param margin display margins (in pixels) + */ + public void show(int width, int height, int margin) { + BufferedImage bim + = new BufferedImage(width + 2 * margin, + height + 2 * margin, + BufferedImage.TYPE_INT_ARGB); + Graphics2D g = bim.createGraphics(); + double xhigh = Arrays.max(X); + double xlow = Arrays.min(X); + double xrange = xhigh - xlow; + double yhigh = Arrays.max(Y); + double ylow = 0; //ArrayMath.min(Y); + double yrange = yhigh - ylow; + + // draw bars + g.setColor(Color.RED); + for (int n = 0; n < X.length; ++n) { + int xpos = (int)((width * (X[n] - xlow)) / xrange); + int ypos = (int)((height * (Y[n] - ylow)) / yrange); + g.fillRect(margin + xpos - 1, height + margin - ypos, 3, ypos); + } + + // draw X and Y axes + g.setColor(Color.BLUE); + g.drawRect(margin, margin, width, height); + + // draw Y-tics + int e = (int) Math.ceil(Math.log(yrange) / Math.log(10)) - 1; + int ystep = (e > 0) ? pow(10, e) : 1; + for (int y = ystep * (int)(ylow/ystep); y <= yhigh; y += ystep) { + int ypos = (int)((height * (y - ylow)) / yrange); + g.drawString(String.valueOf(y) + "-", 0, height + margin - ypos); + } + + // draw X-tics + e = (int) Math.ceil(Math.log(xrange) / Math.log(10)) - 1; + int xstep = (e > 0) ? pow(10, e) : 1; + for (int x = xstep * (int)(xlow/xstep); x <= xhigh; x += xstep) { + int xpos = (int)((width * (x - xlow)) / xrange); + g.drawString(String.valueOf(x) , margin + xpos - 6 * e, height + margin + 12); + g.drawLine(margin + xpos, height + margin, + margin + xpos, height + margin - 5); + } + + g.dispose(); + System.out.println("Showing on screen"); + eu.digitisation.image.Display.draw(bim); + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/ngram/ContextLengthRange.java b/ocrevalUAtion/src/main/java/eu/digitisation/ngram/ContextLengthRange.java new file mode 100644 index 00000000..48cdbbbb --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/ngram/ContextLengthRange.java @@ -0,0 +1,45 @@ +package eu.digitisation.ngram; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class ContextLengthRange { + private int start; + + public int getStart() { + return start; + } + + public int getEnd() { + return end; + } + + private int end; + + public ContextLengthRange(int start, int end) { + this.start = start; + this.end = end; + } + + /** + * Creates ContextLengthRange objects based on the provided string + * + * @param contextLengthRange + * context length range string which needs to follow the + * following regex: [1-9]-[1-9] where the first digit is a start + * and the second is end + * @return ContextLengthRange object created from the given string + */ + public static ContextLengthRange parseContextLengthRange( + String contextLengthRange) { + Pattern clPattern = Pattern.compile("([1-9])-([1-9])"); + Matcher clMatcher = clPattern.matcher(contextLengthRange); + if (clMatcher.find()) { + return new ContextLengthRange(Integer.parseInt(clMatcher.group(1)), + Integer.parseInt(clMatcher.group(2))); + } else { + throw new IllegalArgumentException( + "Context length needs to be in format of [1-9]-[1-9]!"); + } + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/ngram/Distance.java b/ocrevalUAtion/src/main/java/eu/digitisation/ngram/Distance.java new file mode 100644 index 00000000..a10ec36d --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/ngram/Distance.java @@ -0,0 +1,105 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.ngram; + +/** + * Several types of distances between n-gram models + * + * @author R.C.C. + */ +public class Distance { + + /** + * + * @param first an NgramModel + * @param second a second NgramModel (its order must be identical to + * first's) + * @return + */ + public static double[] delta(NgramModel first, NgramModel second) { + double[] result = new double[first.order]; + int[] deltas = new int[first.order]; + int[] tot = new int[first.order]; + + if (first.order != second.order) { + throw new IllegalArgumentException("Illegal comparison " + + "of n-gram models with different n"); + } + for (String s : first.occur.keySet()) { + if (s.length() > 0) { + int val1 = first.occur.get(s).getValue(); + int val2 = second.occur.containsKey(s) + ? second.occur.get(s).getValue() : 0; + deltas[s.length() - 1] += Math.abs(val1 - val2); + tot[s.length() - 1] += (val1 + val2); + } + } + for (String s : second.occur.keySet()) { + if (s.length() > 0 && !first.occur.containsKey(s)) { + int val2 = second.occur.get(s).getValue(); + deltas[s.length() - 1] += val2; + tot[s.length() - 1] += val2; + } + } + for (int n = 0; n < first.order; ++n) { + //result += Math.log(deltas[n] / (double) tot[n]); + result[n] = deltas[n] / (double) tot[n]; + } +// return Math.exp(result / first.order); + return result; + } + + /** + * + * @param first an NgramModel + * @param second a second NgramModel (its order must be identical to + * first's) + * @return + */ + public static double cosine(NgramModel first, NgramModel second) { + int sum = 0; + int norm1 = 0; + int norm2 = 0; + + if (first.order != second.order) { + throw new IllegalArgumentException("Illegal comparison " + + "of n-gram models with different n"); + } + for (String s : first.occur.keySet()) { + if (s.length() > 0) { + int val1 = first.occur.get(s).getValue(); + int val2 = second.occur.containsKey(s) + ? second.occur.get(s).getValue() : 0; + sum += val1 * val2; + norm1 += val1 * val1; + norm2 += val2 * val2; + } + } + for (String s : second.occur.keySet()) { + if (s.length() > 0 && !first.occur.containsKey(s)) { + int val2 = second.occur.get(s).getValue(); + norm2 += val2 * val2; + } + } + if (norm1 > 0 && norm2 > 0) { + return sum / (Math.sqrt(norm1) * Math.sqrt(norm2)); + } else { + return 0; + } + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/ngram/EvaluationFrame.java b/ocrevalUAtion/src/main/java/eu/digitisation/ngram/EvaluationFrame.java new file mode 100644 index 00000000..8e8b6360 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/ngram/EvaluationFrame.java @@ -0,0 +1,243 @@ +package eu.digitisation.ngram; + +import java.awt.Color; +import java.awt.EventQueue; +import java.awt.Font; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.util.ArrayList; +import java.util.List; +import javax.swing.GroupLayout; +import javax.swing.GroupLayout.Alignment; +import javax.swing.JFrame; +import javax.swing.JOptionPane; +import javax.swing.JPanel; +import javax.swing.JScrollPane; +import javax.swing.JSlider; +import javax.swing.JTextField; +import javax.swing.JTextPane; +import javax.swing.LayoutStyle.ComponentPlacement; +import javax.swing.border.EmptyBorder; +import javax.swing.event.ChangeEvent; +import javax.swing.event.ChangeListener; +import javax.swing.text.SimpleAttributeSet; +import javax.swing.text.StyleConstants; +import javax.swing.text.StyledDocument; + +public class EvaluationFrame extends JFrame { + + /** + * + */ + private static final long serialVersionUID = 4895806099667768081L; + + private JPanel contentPane; + private JTextField thresholdTextField; + private JSlider thresholdSlider; + private JTextPane textPane; + + /** + * double array containing the perplexity values for every character. + */ + private double[] perplexityArray; + + /** + * text style applied when the threshold value is exceeded. + */ + private static final SimpleAttributeSet thresholdExceededStyle = new SimpleAttributeSet(); + /** + * default text style. + */ + private static final SimpleAttributeSet defaultStyle = new SimpleAttributeSet(); + + static { + StyleConstants.setForeground(thresholdExceededStyle, Color.red); + StyleConstants.setForeground(defaultStyle, Color.black); + } + + /** + * Create the frame. + */ + public EvaluationFrame() { + init(); + } + + /** + * GUI initialization. + */ + private void init() { + setBounds(100, 100, 473, 347); + contentPane = new JPanel(); + contentPane.setBorder(new EmptyBorder(5, 5, 5, 5)); + setContentPane(contentPane); + + JPanel panel = new JPanel(); + + JScrollPane scrollPane = new JScrollPane(); + GroupLayout gl_contentPane = new GroupLayout(contentPane); + gl_contentPane.setHorizontalGroup(gl_contentPane.createParallelGroup(Alignment.LEADING) + .addComponent(panel, GroupLayout.DEFAULT_SIZE, 447, Short.MAX_VALUE) + .addComponent(scrollPane, GroupLayout.DEFAULT_SIZE, 447, Short.MAX_VALUE)); + gl_contentPane.setVerticalGroup(gl_contentPane.createParallelGroup(Alignment.LEADING).addGroup( + gl_contentPane + .createSequentialGroup() + .addComponent(panel, GroupLayout.PREFERRED_SIZE, GroupLayout.DEFAULT_SIZE, + GroupLayout.PREFERRED_SIZE).addPreferredGap(ComponentPlacement.RELATED) + .addComponent(scrollPane, GroupLayout.DEFAULT_SIZE, 240, Short.MAX_VALUE))); + + textPane = new JTextPane(); + textPane.setFont(new Font("Tahoma", Font.PLAIN, 16)); + scrollPane.setViewportView(textPane); + + thresholdTextField = new JTextField(); + thresholdTextField.setEditable(false); + thresholdTextField.setFont(new Font("Tahoma", Font.BOLD, 16)); + thresholdTextField.setText("-1"); + thresholdTextField.setColumns(10); + + thresholdSlider = new JSlider(); + thresholdSlider.setValue(-1); + thresholdSlider.setMaximum(-1); + thresholdSlider.setMinimum(-50); + thresholdSlider.addChangeListener(new ChangeListener() { + + @Override + public void stateChanged(ChangeEvent e) { + if (!thresholdSlider.getValueIsAdjusting()) { + thresholdTextField.setText((double) thresholdSlider.getValue() + ""); + update(true); + } + } + }); + GroupLayout gl_panel = new GroupLayout(panel); + gl_panel.setHorizontalGroup(gl_panel.createParallelGroup(Alignment.TRAILING).addGroup( + gl_panel.createSequentialGroup() + .addComponent(thresholdSlider, GroupLayout.DEFAULT_SIZE, 357, Short.MAX_VALUE) + .addPreferredGap(ComponentPlacement.RELATED) + .addComponent(thresholdTextField, GroupLayout.PREFERRED_SIZE, 74, GroupLayout.PREFERRED_SIZE) + .addContainerGap())); + gl_panel.setVerticalGroup(gl_panel.createParallelGroup(Alignment.TRAILING).addGroup( + gl_panel.createSequentialGroup() + .addContainerGap() + .addGroup( + gl_panel.createParallelGroup(Alignment.TRAILING) + .addComponent(thresholdSlider, Alignment.LEADING, GroupLayout.DEFAULT_SIZE, 31, + Short.MAX_VALUE) + .addComponent(thresholdTextField, Alignment.LEADING, GroupLayout.DEFAULT_SIZE, + 31, Short.MAX_VALUE)).addContainerGap())); + panel.setLayout(gl_panel); + contentPane.setLayout(gl_contentPane); + + } + + /** + * update the evaluation results. + */ + private void update(boolean thresholdMode) { + Double threshold = 0.0; + try { + threshold = Double.parseDouble(thresholdTextField.getText()); + StyledDocument document = textPane.getStyledDocument(); + + document.setCharacterAttributes(0, document.getLength(), defaultStyle, true); + if (thresholdMode) { + for (int i = 0; i < perplexityArray.length; i++) { + if (perplexityArray[i] < threshold) { + document.setCharacterAttributes(i, 1, thresholdExceededStyle, true); + } + } + } else { + double max = 0; + + for (int i = 0; i < perplexityArray.length; i++) { + double value = perplexityArray[i]; + if (!Double.isInfinite(value)) { + if (Math.abs(value) > Math.abs(max)) { + max = value; + } + } + } + + List colors = getColorBands(Color.red, 11); + + for (int i = 0; i < perplexityArray.length; i++) { + SimpleAttributeSet style = new SimpleAttributeSet(); + double value = perplexityArray[i]; + if (Double.isInfinite(value)) { + StyleConstants.setForeground(style, Color.red); + } else { + int color = 10 - (int) ((value / max) * 10); + StyleConstants.setForeground(style, colors.get(color)); + } + document.setCharacterAttributes(i, 1, style, true); + } + } + + } catch (NumberFormatException nfe) { + JOptionPane.showMessageDialog(this, "Unable to parse value '" + + thresholdTextField.getText() + "'", + "Error", JOptionPane.ERROR_MESSAGE); + } + } + + public void setInput(String textToEvaluate, double[] perplexityArray) { + this.perplexityArray = perplexityArray; + textPane.setText(textToEvaluate); + update(false); + } + + public List getColorBands(Color color, int bands) { + + List colorBands = new ArrayList(bands); + for (int index = 0; index < bands; index++) { + colorBands.add(darken(color, index / (double) bands)); + } + return colorBands; + + } + + public static Color darken(Color color, double fraction) { + + int red = (int) Math.round(Math.max(0, color.getRed() - 255 * fraction)); + int green = (int) Math.round(Math.max(0, color.getGreen() - 255 * fraction)); + int blue = (int) Math.round(Math.max(0, color.getBlue() - 255 * fraction)); + + int alpha = color.getAlpha(); + + return new Color(red, green, blue, alpha); + + } + + /** + * Launch the application. + * + * @param args + */ + public static void main(final String[] args) { + + EventQueue.invokeLater(new Runnable() { + @Override + public void run() { + try { + File model = new File(args[0]); + File input = new File(args[1]); + int contextLenght = Integer.parseInt(args[2]); + + TextPerplexity result + = new TextPerplexity(new NgramModel(model), + new FileInputStream(input), contextLenght); + + EvaluationFrame frame = new EvaluationFrame(); + frame.setInput(result.getText(), result.getPerplexities()); + frame.setVisible(true); + + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (NumberFormatException e) { + e.printStackTrace(); + } + } + }); + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/ngram/Experiment.java b/ocrevalUAtion/src/main/java/eu/digitisation/ngram/Experiment.java new file mode 100644 index 00000000..d19c7821 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/ngram/Experiment.java @@ -0,0 +1,102 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, transform to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.ngram; + +import eu.digitisation.input.WarningException; +import eu.digitisation.layout.SortPageXML; +import eu.digitisation.output.ErrorMeasure; +import eu.digitisation.text.CharFilter; +import eu.digitisation.text.Text; +import java.io.File; +import java.io.IOException; + +/** + * + * @author R.C.C. + */ +public class Experiment { + + private static void compare(File f1, File f2, File f3, CharFilter filter) + throws WarningException, IOException { + Text c1 = new Text(f1); + Text c2 = new Text(f2); + Text c3 = new Text(f3); + String s1 = c1.toString(filter); + String s2 = c2.toString(filter); + String s3 = c3.toString(filter); + boolean sorted = SortPageXML.isSorted(f1); + int l1 = s1.length(); + final int N = 10; + + if (l1 < 100) { + System.err.println("Text is too short (" + l1 + " characters)"); + } else { + double cer12 = ErrorMeasure.cer(s1, s2); + double cer32 = ErrorMeasure.cer(s3, s2); + double[] errors; + double cosineDist; + + NgramModel m1 = new NgramModel(N); + NgramModel m2 = new NgramModel(N); + m1.addWord(s1); + m2.addWord(s2); + + cosineDist = 1 - Distance.cosine(m1, m2); + errors = Distance.delta(m1, m2); + + System.out.print(f1.getName() + " " + sorted + + " " + String.format("%05d", s1.length()) + + " " + String.format("%.3f", cer12) + + " " + String.format("%.3f", cer32) + + " " + String.format("%.3f", cosineDist)); + for (int n = 1; n <= N; ++n) { + System.out.print(" " + String.format("%.3f", errors[n - 1])); + } + System.out.println(); + } + } + + public static void main(String[] args) + throws IOException, WarningException { + File dir1 = new File(args[0]); + File dir2 = new File(args[1]); + File dir3 = new File("/tmp"); + CharFilter filter = args.length > 2 + ? new CharFilter(new File(args[2])) + : new CharFilter(true); + if (dir1.isFile()) { + dir3 = new File("/tmp", + dir1.getName().replace(".xml", "_sorted.xml")); + + if (dir2.exists()) { + SortPageXML.transform(dir1, dir3); + Experiment.compare(dir1, dir2, dir3, filter); + } + } else { + for (File f1 : dir1.listFiles()) { + String name = f1.getName(); + File f2 = new File(dir2, name.replace(".xml", ".html")); + File f3 = new File(dir3, name.replace(".xml", "_sorted.xml")); + if (f2.exists()) { + SortPageXML.transform(f1, f3); + compare(f1, f2, f3, filter); + } + } + } + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/ngram/Int.java b/ocrevalUAtion/src/main/java/eu/digitisation/ngram/Int.java new file mode 100644 index 00000000..2b68329b --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/ngram/Int.java @@ -0,0 +1,196 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.ngram; + +import java.io.Serializable; + +/** + * A mutable Integer (faster than boxing/un-boxing) + * + * @author Rafael C. Carrasco + * @version 1.1 + */ +public class Int implements Comparable, Serializable { + private static final long serialVersionUID = 1L; + + int n; // The value. + + /** + * Constructs a new Int with the specified int value. + * + * @param n the integer value + */ + public Int(int n) { + this.n = n; + } + + /** + * Constructs a new Int with the value indicated by the string. + * + * @param s string representing the integer value + */ + public Int(String s) { + n = Integer.parseInt(s); + } + + /** + * Returns the value as int. + * + * @return he value as int + */ + public int getValue() { + return n; + } + + /** + * Assigns the specified int value. + * + * @param n the value for this Int + * @return the Int itself + */ + public Int setValue(int n) { + this.n = n; + return this; + } + + /** + * Pre-increments value by 1. + * + * @return the Int itself + */ + public Int increment() { + ++n; + return this; + } + + /** + * Add to value. + * + * @param n the delta value + * @return he Int itself + */ + public Int add(int n) { + this.n += n; + return this; + } + + /** + * Add to value. + * + * @param n the delta value + * @return he Int itself + */ + public Int subtact(int n) { + this.n -= n; + return this; + } + + /** + * Pre-decrements value by 1. + * + * @return he Int itself + */ + public Int decrement() { + --n; + return this; + } + + /** + * Post-increments value by one. + * + * @return he Int itself + */ + public Int postIncValue() { + ++n; + return new Int(n - 1); + } + + /** + * Returns a new Int with incremented value. + * + * @return he Int itself + */ + public Int nextInt() { + return new Int(n + 1); + } + + /** + * Returns a String object representing the specified Int. + */ + @Override + public String toString() { + return String.valueOf(n); + } + + /** + * Tests if two Int objects store the same value. + * @param other another Int object + * @return true if values are identical + */ + public boolean equals(Int other) { + return this.n == other.n; + } + + /** + * Tests if this Int objects stores a given value. + * @param n an integer value + * @return true if his Int objects stores n + */ + public boolean equals(int n) { + return this.n == n; + } + + /** + * Compares this object to the specified object. The result is true if and + * only if the argument is not null and is an Int object that contains the + * same int value as this object. + * @param object another pair + */ + @Override + public boolean equals(Object object) { + if (object == null) { + return false; + } else if (this == object) { + return true; + } + if (object instanceof Int) { + return this.n == ((Int) object).n; + } else { + return false; + } + + } + + @Override + public int hashCode() { + return n; + } + + /** + * Compares two Int objects numerically. + * @param N another Int object + */ + @Override + public int compareTo(Int N) { + if (n < N.n) { + return -1; + } else { + return (n == N.n) ? 0 : 1; + } + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/ngram/MNgramModel.tmp b/ocrevalUAtion/src/main/java/eu/digitisation/ngram/MNgramModel.tmp new file mode 100644 index 00000000..7e809f14 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/ngram/MNgramModel.tmp @@ -0,0 +1,513 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.ngram; + +import eu.digitisation.log.Messages; +import eu.digitisation.text.WordScanner; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; +import java.nio.charset.Charset; +import java.util.HashMap; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +/** + * A n-gram model for strings. N is the maximal order of the model (context + * length plus one). + */ +public class NgramModel implements Serializable { + + static final long serialVersionUID = 1L; + static final String BOS = "\u0002"; // Begin of string text marker. + static final String EOS = "\u0003"; // End of text marker. + int order; // The size of the context plus one (n-gram). + HashMap occur; // Number of occurrences. + double[] lambda; // Backoff parameters + + /** + * Set maximal order of model. + * + * @param order the size of the context plus one (the n in n-gram). + */ + public final void setOrder(int order) { + if (occur == null || occur.isEmpty()) { + if (order > 0) { + this.order = order; + } else { + throw new IllegalArgumentException("Order must be grater than 0"); + } + } else { + throw new IllegalStateException("Cannot change order of model with previous content"); + } + } + + /** + * Class constructor (default order is 2). + */ + public NgramModel() { + setOrder(2); + occur = new HashMap(); + lambda = null; + + } + + /** + * Class constructor. + * + * @param order the size of the context plus one. + */ + public NgramModel(int order) { + setOrder(order); + occur = new HashMap(); + lambda = null; + } + + /** + * @return number of different n-grams stored + */ + public int size() { + return occur.keySet().size(); + } + + /** + * Save n-gram model to GZIP file + * + * @param file the output file + */ + public void save(File file) { + try { + FileOutputStream fos = new FileOutputStream(file); + GZIPOutputStream gos = new GZIPOutputStream(fos); + ObjectOutputStream out = new ObjectOutputStream(gos); + out.writeObject(this); + out.close(); + } catch (IOException ex) { + Messages.info(NgramModel.class.getName() + ": " + ex); + } + } + + /** + * Build n-gram model from file + * + * @param file the GZIP input file + */ + public NgramModel(File file) { + try { + FileInputStream fis = new FileInputStream(file); + GZIPInputStream gis = new GZIPInputStream(fis); + ObjectInputStream in = new ObjectInputStream(gis); + NgramModel ngram = (NgramModel) in.readObject(); + in.close(); + this.order = ngram.order; + this.occur = ngram.occur; + this.lambda = ngram.lambda; + } catch (IOException ex) { + Messages.info(NgramModel.class.getName() + ": " + ex); + } catch (ClassNotFoundException ex) { + Messages.info(NgramModel.class.getName() + ": " + ex); + } + } + + /** + * @return Good-Turing back-off parameters. + */ + public double[] getGoodTuringPars() { + double[] pars = new double[order]; + int[] total = new int[order]; + int[] singles = new int[order]; + for (String word : occur.keySet()) { + if (word.length() > 0) { + int k = word.length() - 1; + int times = occur.get(word).getValue(); + total[k] += times; + if (times == 1) { + ++singles[k]; + } + } + } + for (int k = 0; k < order; ++k) { + pars[k] = singles[k] / (double) total[k]; + } + return pars; + } + + /** + * @param n n-gram order. + * @return Good-Turing back-off parameter. + */ + private double lambda(int n) { + if (lambda == null) { + lambda = getGoodTuringPars(); + } + return lambda[n]; + } + + /** + * @param s a k-gram + * @return the (k-1)-gram obtained by removing its first character. + */ + private String tail(String s) { + return s.substring(1); + } + + /** + * @param s a k-gram + * @return the (k-1)-gram obtained by removing its last character. + */ + private String head(String s) { + return s.substring(0, s.length() - 1); + } + + /** + * @return the number of text entries (usually words) building the model. + */ + private int numWords() { + return occur.get(EOS).getValue(); // end-of-word. + } + + /** + * @param s a k-gram (k > 0) + * @return the conditional probability of the k-gram, normalized to the + * number of heads. + */ + public double prob(String s) { + if (occur.containsKey(s)) { + String h = head(s); + if (h.endsWith(BOS)) { // since head is not stored + return occur.get(s).getValue() / (double) numWords(); + } else { + return occur.get(s).getValue() + / (double) occur.get(h).getValue(); + } + } else { + return 0; + } + } + + /** + * @param s a k-gram + * @return the conditional probability of the k-gram, normalized to the + * frequency of its heads and interpolated with lower order models. + */ + public double smoothProb(String s) { + double result; + if (s.length() > 1) { + double lam = lambda(s.length() - 1); + result = (1 - lam) * prob(s) + lam * smoothProb(tail(s)); + } else { + result = prob(s); + } + return result; + } + + /** + * @param s a k-gram + * @return the expected number of occurrences (per word) of s. + */ + private double expectedNumberOf(String s) { + if (s.endsWith(BOS)) { + return 1; + } else { + return occur.get(s).getValue() / (double) numWords(); + } + } + + /** + * Increments number of occurrences of s. + * + * @param s a k-gram. + */ + private void addEntry(String s) { + if (occur.containsKey(s)) { + occur.get(s).increment(); + } else { + occur.put(s, new Int(1)); + } + } + + /** + * Increments number of occurrences of s. + * + * @param s a k-gram. + * @param n number of occurrences + */ + private void addEntries(String s, int n) { + if (occur.containsKey(s)) { + occur.get(s).add(n); + } else { + occur.put(s, new Int(n)); + } + } + + /** + * Compute n-gram model log entropy per word (in bits). + * + * @return log entropy per word (in bits). + */ + public double entropy() { + double p, sum = 0; + for (String s : occur.keySet()) { + if (s.length() == order) { + p = prob(s); + sum -= expectedNumberOf(head(s)) * p * Math.log(p); + } + } + return sum / Math.log(2); + } + + /** + * Extracts all k-grams in a word or text upto the maximal order. For + * instance, if word = "ma" and order = 3, then 0-grams are: "" (three empty + * strings, used to normalize 1-grams); three uni-grams: "m, a, $" ($ + * represents end-of-string); three bi-grams: "#m, ma, a$" (# is used to + * differentiate #m from 1-gram m); and three tri-grams: "##m, #ma, ma$" + * + * @remark never add uni-gram "#" to the model because the normalization of + * uni-grams will be wrong! + * @param word the word or text (string of characters) to be added. + */ + public void add(String word) { + if (word.length() < 1) { + throw new IllegalStateException("Cannot extract n-grams from empty word"); + } else { + word += EOS; + } + String s = ""; + while (s.length() < order) { + s += BOS; + } + for (int last = 0; last < word.length(); ++last) { + s = tail(s) + word.charAt(last); + for (int first = 0; first <= s.length(); ++first) { + addEntry(s.substring(first)); + } + } + } + + /** + * Add all k-grams in a word or text + * + * @param word the word or text to be processed + * @param times the number of occurrences of the word or text + */ + public void add(String word, int times) { + if (word.length() < 1) { + throw new IllegalStateException("Cannot extract n-grams from empty word"); + } else { + word += EOS; + } + String s = ""; + while (s.length() < order) { + s += BOS; + } + for (int last = 0; last < word.length(); ++last) { + s = tail(s) + word.charAt(last); + for (int first = 0; first <= s.length(); ++first) { + addEntries(s.substring(first), times); + } + } + } + + /** + * Reads text file and adds words to model. + * + * @param file a text file + * @param encoding the text encoding + * @param caseSensitive true if extracted n-grams are case sensitive + */ + public void addWords(File file, Charset encoding, boolean caseSensitive) { + try { + WordScanner scanner = new WordScanner(file, encoding, "^\\p{Space}+"); + String word; + while ((word = scanner.nextWord()) != null) { + if (caseSensitive) { + add(word); + } else { + add(word.toLowerCase()); + + } + } + } catch (IOException ex) { + Messages.info(NgramModel.class + .getName() + ": " + ex); + } + } + + /** + * Compute probability of a word or text + * + * @param text a word or a sequence of characters + * @return the log-probability (base e) of the sequence + */ + public double logProb(String text) { + double res = 0; + if (text.length() < 1) { + throw new IllegalArgumentException("Cannot compute probability of empty word"); + } else { + text += EOS; + } + String s = ""; + while (s.length() < order) { + s += BOS; + } + for (int last = 0; last < text.length(); ++last) { + s = tail(s) + text.charAt(last); + + double p = smoothProb(s); + if (p == 0) { + System.err.println(s + " has 0 probability"); + return Double.NEGATIVE_INFINITY; + } else { + res += Math.log(p); + } + } + return res; + } + + /** + * Compute probability of a character after a given context. This + * implementation only takes into account the preceding context + * + * @param context a sequence of characters + * @param c a character + * @return the log-probability (base e) that the character c follows the + * given context + */ + public double logProb(String context, char c) { + double res = 0; + int len = context.length() + 1; // the length of the context + character + String s = len > order + ? context.substring(len - order) + c + : context + c; + double p = smoothProb(s); + + return (p > 0) + ? Math.log(p) + : Double.NEGATIVE_INFINITY; + } + + /** + * Reads input text and computes per-word cross entropy. + * + * @param caseSensitive true if the model is case sensitive + * @return the log-likelihood of text 8per word). + */ + public double logPerWordLikelihood(boolean caseSensitive) { + try { + Charset encoding = Charset.forName(System.getProperty("file.encoding")); + WordScanner scanner = new WordScanner(System.in, encoding); + String word; + double result = 0; + int numWords = 0; + while ((word = scanner.nextWord()) != null) { + ++numWords; + if (caseSensitive) { + result -= logProb(word); + } else { + result -= logProb(word.toLowerCase()); + } + } + + return result / numWords / Math.log(2); + + } catch (IOException ex) { + Messages.info(NgramModel.class + .getName() + ": " + ex); + } + return Double.POSITIVE_INFINITY; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + for (String s : occur.keySet()) { + builder.append(s.replaceAll(BOS, "").replaceAll(EOS, "")); + builder.append(' ').append(occur.get(s)).append('\n'); + } + return builder.toString(); + } + + /** + * Show differences between model (debug function) + * + * @param other another NgramModel (order must coincide) + */ + public void showDiff(NgramModel other) { + if (this.order != other.order) { + throw new IllegalArgumentException("Illegal comparison " + + "of n-gram models with different n"); + } + for (String s : this.occur.keySet()) { + if (s.length() > 0) { + int val1 = this.occur.get(s).getValue(); + int val2 = other.occur.containsKey(s) + ? other.occur.get(s).getValue() : 0; + if (val1 != val2) { + System.out.println(s + " " + val1 + " " + val2); + } + } + } + for (String s : other.occur.keySet()) { + if (s.length() > 0 && !this.occur.containsKey(s)) { + int val2 = other.occur.get(s).getValue(); + System.out.println(s + " 0 " + val2); + } + } + } + + /** + * Main function. + * + * @param args + */ + public static void main(String[] args) { + NgramModel ngram = new NgramModel(); + Charset encoding = Charset.forName(System.getProperty("file.encoding")); + File fout = null; + + if (args.length == 0) { + System.err.println("Usage: Ngram [-n n] [-e encoding] [-o outfile]" + + " file1 file2 ...."); + } else { + for (int k = 0; k < args.length; ++k) { + String arg = args[k]; + + if (arg.equals("-n")) { + ngram.setOrder(new Integer(args[++k])); + } else if (arg.equals("-e")) { + encoding = Charset.forName(args[++k]); + } else if (arg.equals("-o")) { + fout = new File(args[++k]); + } else { + ngram.addWords(new File(arg), encoding, false); + } + } + if (fout != null) { + ngram.save(fout); + } else { + System.out.println(ngram.entropy()); + System.out.println(ngram.logPerWordLikelihood(false)); + } + } + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/ngram/MNgramModel.txt b/ocrevalUAtion/src/main/java/eu/digitisation/ngram/MNgramModel.txt new file mode 100644 index 00000000..01bdcf38 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/ngram/MNgramModel.txt @@ -0,0 +1,513 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.ngram; + +import eu.digitisation.log.Messages; +import eu.digitisation.text.WordScanner; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; +import java.util.HashMap; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +/** + * A n-gram model for strings. N is the maximal order of the model (context + * length plus one). + */ +public class NgramModel implements Serializable { + + static final long serialVersionUID = 1L; + static final String BOS = "\u0002"; // Begin of string text marker. + static final String EOS = "\u0003"; // End of text marker. + int order; // The size of the context plus one (n-gram). + HashMap occur; // Number of occurrences. + double[] lambda; // Backoff parameters + + /** + * Set maximal order of model. + * + * @param order the size of the context plus one (the n in n-gram). + */ + public final void setOrder(int order) { + if (occur == null || occur.isEmpty()) { + if (order > 0) { + this.order = order; + } else { + throw new IllegalArgumentException("Order must be grater than 0"); + } + } else { + throw new IllegalStateException("Cannot change order of model with previous content"); + } + } + + /** + * Class constructor (default order is 2). + */ + public NgramModel() { + setOrder(2); + occur = new HashMap(); + lambda = null; + + } + + /** + * Class constructor. + * + * @param order the size of the context plus one. + */ + public NgramModel(int order) { + setOrder(order); + occur = new HashMap(); + lambda = null; + } + + /** + * @return number of different n-grams stored + */ + public int size() { + return occur.keySet().size(); + } + + /** + * Save n-gram model to GZIP file + * + * @param file the output file + */ + public void save(File file) { + try { + FileOutputStream fos = new FileOutputStream(file); + GZIPOutputStream gos = new GZIPOutputStream(fos); + ObjectOutputStream out = new ObjectOutputStream(gos); + out.writeObject(this); + out.close(); + } catch (IOException ex) { + Messages.info(NgramModel.class.getName() + ": " + ex); + } + } + + /** + * Build n-gram model from file + * + * @param file the GZIP input file + */ + public NgramModel(File file) { + try { + FileInputStream fis = new FileInputStream(file); + GZIPInputStream gis = new GZIPInputStream(fis); + ObjectInputStream in = new ObjectInputStream(gis); + NgramModel ngram = (NgramModel) in.readObject(); + in.close(); + this.order = ngram.order; + this.occur = ngram.occur; + this.lambda = ngram.lambda; + } catch (IOException ex) { + Messages.info(NgramModel.class.getName() + ": " + ex); + } catch (ClassNotFoundException ex) { + Messages.info(NgramModel.class.getName() + ": " + ex); + } + } + + /** + * @return Good-Turing back-off parameters. + */ + public double[] getGoodTuringPars() { + double[] pars = new double[order]; + int[] total = new int[order]; + int[] singles = new int[order]; + for (String word : occur.keySet()) { + if (word.length() > 0) { + int k = word.length() - 1; + int times = occur.get(word).getValue(); + total[k] += times; + if (times == 1) { + ++singles[k]; + } + } + } + for (int k = 0; k < order; ++k) { + pars[k] = singles[k] / (double) total[k]; + } + return pars; + } + + /** + * @param n n-gram order. + * @return Good-Turing back-off parameter. + */ + private double lambda(int n) { + if (lambda == null) { + lambda = getGoodTuringPars(); + } + return lambda[n]; + } + + /** + * @param s a k-gram + * @return the (k-1)-gram obtained by removing its first character. + */ + private String tail(String s) { + return s.substring(1); + } + + /** + * @param s a k-gram + * @return the (k-1)-gram obtained by removing its last character. + */ + private String head(String s) { + return s.substring(0, s.length() - 1); + } + + /** + * @return the number of text entries (usually words) building the model. + */ + private int numWords() { + return occur.get(EOS).getValue(); // end-of-word. + } + + /** + * @param s a k-gram (k > 0) + * @return the conditional probability of the k-gram, normalized to the + * number of heads. + */ + public double prob(String s) { + if (occur.containsKey(s)) { + String h = head(s); + if (h.endsWith(BOS)) { // since head is not stored + return occur.get(s).getValue() / (double) numWords(); + } else { + return occur.get(s).getValue() + / (double) occur.get(h).getValue(); + } + } else { + return 0; + } + } + + /** + * @param s a k-gram + * @return the conditional probability of the k-gram, normalized to the + * frequency of its heads and interpolated with lower order models. + */ + public double smoothProb(String s) { + double result; + if (s.length() > 1) { + double lam = lambda(s.length() - 1); + result = (1 - lam) * prob(s) + lam * smoothProb(tail(s)); + } else { + result = prob(s); + } + return result; + } + + /** + * @param s a k-gram + * @return the expected number of occurrences (per word) of s. + */ + private double expectedNumberOf(String s) { + if (s.endsWith(BOS)) { + return 1; + } else { + return occur.get(s).getValue() / (double) numWords(); + } + } + + /** + * Increments number of occurrences of s. + * + * @param s a k-gram. + */ + private void addEntry(String s) { + if (occur.containsKey(s)) { + occur.get(s).increment(); + } else { + occur.put(s, new Int(1)); + } + } + + /** + * Increments number of occurrences of s. + * + * @param s a k-gram. + * @param n number of occurrences + */ + private void addEntries(String s, int n) { + if (occur.containsKey(s)) { + occur.get(s).add(n); + } else { + occur.put(s, new Int(n)); + } + } + + /** + * Compute n-gram model log entropy per word (in bits). + * + * @return log entropy per word (in bits). + */ + public double entropy() { + double p, sum = 0; + for (String s : occur.keySet()) { + if (s.length() == order) { + p = prob(s); + sum -= expectedNumberOf(head(s)) * p * Math.log(p); + } + } + return sum / Math.log(2); + } + + /** + * Extracts all k-grams in a word or text upto the maximal order. For + * instance, if word = "ma" and order = 3, then 0-grams are: "" (three empty + * strings, used to normalize 1-grams); three uni-grams: "m, a, $" ($ + * represents end-of-string). three bi-grams: "#m, ma, a$" (# is used to + * differentiate #m from 1-gram m); and three tri-grams: "##m, #ma, ma$" + * + * @remark never add uni-gram "#" to the model because the normalization of + * uni-grams will be wrong! + * @param word the word (string of characters) to be added. + */ + public void addWord(String word) { + if (word.length() < 1) { + throw new IllegalStateException("Cannot extract n-grams from empty word"); + } else { + word += EOS; + } + String s = ""; + while (s.length() < order) { + s += BOS; + } + for (int last = 0; last < word.length(); ++last) { + s = tail(s) + word.charAt(last); + for (int first = 0; first <= s.length(); ++first) { + addEntry(s.substring(first)); + } + } + } + + /** + * Add all k-grams in a word or text + * + * @param word the word or text to be processed + * @param times the number of occurrences of the word or text + */ + public void addWords(String word, int times) { + if (word.length() < 1) { + throw new IllegalStateException("Cannot extract n-grams from empty word"); + } else { + word += EOS; + } + String s = ""; + while (s.length() < order) { + s += BOS; + } + for (int last = 0; last < word.length(); ++last) { + s = tail(s) + word.charAt(last); + for (int first = 0; first <= s.length(); ++first) { + addEntries(s.substring(first), times); + } + } + } + + /** + * Reads text file and adds words to model. + * + * @param file a text file + * @param encoding the text encoding + * @param caseSensitive true if extracted n-grams are case sensitive + */ + public void addTextFile(File file, String encoding, boolean caseSensitive) { + try { + WordScanner scanner = new WordScanner(file, encoding); + String word; + while ((word = scanner.nextWord()) != null) { + if (caseSensitive) { + addWord(word); + } else { + addWord(word.toLowerCase()); + + } + } + } catch (IOException ex) { + Messages.info(NgramModel.class + .getName() + ": " + ex); + } + + } + + /** + * Compute probability of a word. + * + * @param word a word. + * @return the log-probability (base e) of the contained n-grams. + */ + public double wordLogProb(String word) { + double res = 0; + if (word.length() < 1) { + throw new IllegalArgumentException("Cannot compute probability of empty word"); + } else { + word += EOS; + } + String s = ""; + while (s.length() < order) { + s += BOS; + } + for (int last = 0; last < word.length(); ++last) { + s = tail(s) + word.charAt(last); + + double p = smoothProb(s); + if (p == 0) { + System.err.println(s + " has 0 probability"); + return Double.NEGATIVE_INFINITY; + } else { + res += Math.log(p); + } + } + return res; + } + + /** + * Compute probability of a character after a given context. This + * implementation only takes into account the preceding context + * + * @param context a sequence of characters + * @param c a character + * @return the log-probability (base e) that the character c follows the + * given context + */ + public double logProb(String context, char c) { + double res = 0; + int len = context.length() + 1; // the length of the context + character + String s = len > order + ? context.substring(len - order) + c + : context + c; + double p = smoothProb(s); + + return (p > 0) + ? Math.log(p) + : Double.NEGATIVE_INFINITY; + } + + /** + * Reads input text and computes cross entropy. + * + * @param caseSensitive true if the model is case sensitive + * @return the log-likelihood of text. + */ + public double logLikelihood(boolean caseSensitive) { + try { + String encoding = System.getProperty("file.encoding"); + WordScanner scanner = new WordScanner(System.in, encoding); + String word; + double result = 0; + int numWords = 0; + while ((word = scanner.nextWord()) != null) { + ++numWords; + if (caseSensitive) { + result -= wordLogProb(word); + } else { + result -= wordLogProb(word.toLowerCase()); + } + } + + return result / numWords / Math.log(2); + + } catch (IOException ex) { + Messages.info(NgramModel.class + .getName() + ": " + ex); + } + return Double.POSITIVE_INFINITY; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + for (String s : occur.keySet()) { + builder.append(s.replaceAll(BOS, "").replaceAll(EOS, "")); + builder.append(' ').append(occur.get(s)).append('\n'); + } + return builder.toString(); + } + + /** + * Show differences between model (debug function) + * + * @param other another NgramModel (order must coincide) + */ + public void showDiff(NgramModel other) { + if (this.order != other.order) { + throw new IllegalArgumentException("Illegal comparison " + + "of n-gram models with different n"); + } + for (String s : this.occur.keySet()) { + if (s.length() > 0) { + int val1 = this.occur.get(s).getValue(); + int val2 = other.occur.containsKey(s) + ? other.occur.get(s).getValue() : 0; + if (val1 != val2) { + System.out.println(s + " " + val1 + " " + val2); + } + } + } + for (String s : other.occur.keySet()) { + if (s.length() > 0 && !this.occur.containsKey(s)) { + int val2 = other.occur.get(s).getValue(); + System.out.println(s + " 0 " + val2); + } + } + } + + /** + * Main function. + * + * @param args + */ + public static void main(String[] args) { + NgramModel ngram = new NgramModel(); + String encoding = System.getProperty("file.encoding"); + File fout = null; + + if (args.length == 0) { + System.err.println("Usage: Ngram [-n n] [-e encoding] [-o outfile]" + + " file1 file2 ...."); + } else { + for (int k = 0; k < args.length; ++k) { + String arg = args[k]; + + if (arg.equals("-n")) { + ngram.setOrder(new Integer(args[++k])); + } else if (arg.equals("-e")) { + encoding = args[++k]; + } else if (arg.equals("-o")) { + fout = new File(args[++k]); + } else { + ngram.addTextFile(new File(arg), encoding, false); + } + } + if (fout != null) { + ngram.save(fout); + } else { + System.out.println(ngram.entropy()); + System.out.println(ngram.logLikelihood(false)); + } + } + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/ngram/NgramModel.java b/ocrevalUAtion/src/main/java/eu/digitisation/ngram/NgramModel.java new file mode 100644 index 00000000..ee4aac8d --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/ngram/NgramModel.java @@ -0,0 +1,650 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.ngram; + +import eu.digitisation.log.Messages; +import eu.digitisation.text.StringNormalizer; +import eu.digitisation.text.WordScanner; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; +import java.nio.charset.Charset; +import java.util.HashMap; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +/** + * A n-gram model for strings. N is the maximal order of the model (context + * length plus one). + */ +public class NgramModel implements Serializable { + + static final long serialVersionUID = 1L; + static final char BOS = '\u0002'; // Begin of string text marker. + static final char EOS = '\u0003'; // End of text marker. + int order; // The size of the context plus one (n-gram). + HashMap occur; // Number of occurrences. + double[] lambda; // Backoff parameters + + /** + * Class constructor. + * + * @param order the size of the context plus one. + */ + public NgramModel(int order) { + if (order > 0) { + this.order = order; + } else { + throw new IllegalArgumentException("N-gram Order must be grater than 0"); + } + occur = new HashMap(); + lambda = null; + } + + /** + * @return number of different n-grams stored + */ + public int size() { + return occur.keySet().size(); + } + + /** + * Save n-gram model to GZIP file + * + * @param file the output file + */ + public void save(File file) { + try { + FileOutputStream fos = new FileOutputStream(file); + GZIPOutputStream gos = new GZIPOutputStream(fos); + ObjectOutputStream out = new ObjectOutputStream(gos); + + out.writeObject(this); + out.close(); + } catch (IOException ex) { + Messages.severe(NgramModel.class.getName() + ": " + ex); + } + } + + /** + * Build n-gram model from file + * + * @param file the GZIP input file + */ + public NgramModel(File file) { + try { + FileInputStream fis = new FileInputStream(file); + GZIPInputStream gis = new GZIPInputStream(fis); + ObjectInputStream in = new ObjectInputStream(gis); + NgramModel ngram = (NgramModel) in.readObject(); + + this.order = ngram.order; + this.occur = ngram.occur; + this.lambda = ngram.lambda; + in.close(); + + Messages.info("Read " + order + "-gram model"); + } catch (IOException ex) { + Messages.severe(NgramModel.class.getName() + ": " + ex); + } catch (ClassNotFoundException ex) { + Messages.severe(NgramModel.class.getName() + ": " + ex); + } + } + + /** + * @return Good-Turing back-off parameters. + */ + public double[] getGoodTuringPars() { + double[] pars = new double[order]; + int[] total = new int[order]; + int[] singles = new int[order]; + for (String word : occur.keySet()) { + if (word.length() > 0) { + int k = word.length() - 1; + int times = occur.get(word).getValue(); + total[k] += times; + if (times == 1) { + ++singles[k]; + } + } + } + for (int k = 0; k < order; ++k) { + pars[k] = singles[k] / (double) total[k]; + } + return pars; + } + + /** + * @param n n-gram order. + * @return Good-Turing back-off parameter. + */ + private double lambda(int n) { + if (lambda == null) { + lambda = getGoodTuringPars(); + } + return lambda[n]; + } + + /** + * @param s a string + * @return the substring obtained by removing its first character. + */ + private String tail(String s) { + return s.substring(1); + } + + /** + * @param s a string + * @return the substring obtained by removing its last character. + */ + private String head(String s) { + return s.substring(0, s.length() - 1); + } + + /** + * + * @param s a string + * @return the last character in the string + */ + private char lastChar(String s) { + return s.charAt(s.length() - 1); + } + + /** + * @return the number of strings in the sample used to build the model. + */ + private int sampleSize() { + return occur.get(String.valueOf(EOS)).getValue(); + } + + public int occurrences(String key) { + if (occur.containsKey(key)) { + return occur.get(key).getValue(); + } else { + return 0; + } + } + + /** + * @param s a non-empty string + * @return the conditional probability of the string relative to the + * probability of its head. + */ + protected double prob(String s) { + double result; + if (s.charAt(0) == BOS) { + result = occurrences(String.valueOf(BOS) + lastChar(s)) + / (double) sampleSize(); + } else if (!occur.containsKey(s)) { + result = 0; + } else { + result = occurrences(s) / (double) occurrences(head(s)); + } + return result; + } + + /** + * @param s a a non-empty string + * @return the conditional probability of the string relative to its head, + * and interpolated with lower-order models. + */ + protected double smoothProb(String s) { + double result; + + if (s.length() > 1) { + if (s.charAt(0) == BOS && s.length() > 2) { + result = smoothProb(s.substring(s.length() - 2, s.length())); + } else { + double lam = lambda(s.length() - 1); + result = (1 - lam) * prob(s) + lam * smoothProb(tail(s)); + } + } else { + result = prob(s); + } + return result; + } + + /** + * Increments number of occurrences of the given string. + * + * @param s a string + */ + protected void addEntry(String s) { + if (occur.containsKey(s)) { + occur.get(s).increment(); + } else { + occur.put(s, new Int(1)); + } + } + + /** + * Increments number of occurrences of s. + * + * @param s a k-gram. + * @param n number of occurrences + */ + protected void addEntries(String s, int n) { + if (occur.containsKey(s)) { + occur.get(s).add(n); + } else { + occur.put(s, new Int(n)); + } + } + + /** + * Extracts all k-grams in a word or text upto the maximal order. For + * instance, if word = "ma" and order = 3, then 0-grams are: "" (three empty + * strings, used to normalize 1-grams); three uni-grams: "m, a, $" ($ + * represents end-of-string); three bi-grams: "#m, ma, a$" (# is used to + * differentiate #m from 1-gram m); and two tri-grams: "#ma, ma$" + * + * @remark It does not add uni-grams "#" to the model since they can never + * appear in the middle of a word. Normalization of bi-grams starting with # + * will use n($) instead, since n(#)=n($) + * + * @param word the word or text (string of characters) to be added. + */ + public void addWord(String word) { + if (word.length() < 1) { + throw new IllegalArgumentException("Cannot extract n-grams from empty word"); + } else { + String input = BOS + word + EOS; + for (int high = 2; high <= input.length(); ++high) { + for (int low = Math.max(0, high - order); low < high; ++low) { + String s = input.substring(low, high); + addEntry(s); + } + } + addEntries("", word.length() + 1); + } + } + + /** + * Reads text file and adds the words in text to model. + * + * @param file a text file + * @param encoding the text encoding + * @param caseSensitive true if extracted n-grams are case sensitive + */ + public void addWords(File file, Charset encoding, boolean caseSensitive) { + try { + WordScanner scanner = new WordScanner(file, encoding, "^\\p{Space}+"); + String word; + while ((word = scanner.nextWord()) != null) { + if (caseSensitive) { + addWord(word); + } else { + addWord(word.toLowerCase()); + } + } + } catch (IOException ex) { + Messages.info(NgramModel.class + .getName() + ": " + ex); + } + } + + /** + * Compute probability of a word or text + * + * @param word a non-empty a sequence of characters + * @return the log-probability (base e) of this string + */ + public double logWordProb(String word) { + double res = 0; + + if (word.length() < 1) { + throw new IllegalArgumentException("Cannot compute probability of empty word"); + } else { + String input = BOS + word + EOS; + for (int high = 2; high <= input.length(); ++high) { + int low = Math.max(0, high - order); + String s = input.substring(low, high); + double p = smoothProb(s); + if (p == 0) { + Messages.warning(s + " has 0 probability"); + return Double.NEGATIVE_INFINITY; + } else { + res += Math.log(p); + } + } + } + return res; + } + + /** + * Reads input text from standard input and computes per-word cross entropy. + * + * @param caseSensitive true if the model is case sensitive + * @return the log-likelihood of input text (per word). + */ + public double logPerWordLikelihood(boolean caseSensitive) { + try { + Charset encoding = Charset.forName(System.getProperty("file.encoding")); + WordScanner scanner = new WordScanner(System.in, encoding); + + String word; + double result = 0; + int numWords = 0; + while ((word = scanner.nextWord()) != null) { + ++numWords; + if (caseSensitive) { + result -= logWordProb(word); + } else { + result -= logWordProb(word.toLowerCase()); + } + } + + return result / numWords / Math.log(2); + + } catch (IOException ex) { + Messages.severe(NgramModel.class + .getName() + ": " + ex); + } + return Double.POSITIVE_INFINITY; + } + + /** + * Add all the content in a text file + * + * @param is the input stream with text content + */ + public void addText(InputStream is) { + try { + BufferedReader reader + = new BufferedReader(new InputStreamReader(is)); + String context = String.valueOf(BOS); + + while (reader.ready()) { + String line = StringNormalizer.reduceWS(reader.readLine()); + if (!line.isEmpty()) { + String input + = (context.charAt(0) == BOS) + ? line + : " " + line; + addSubstrings(context, input); + if (input.length() >= order) { + context = input.substring(input.length() - order + 1); + } else { + String s = context + input; + context = s.substring(Math.max(0, s.length() - order + 1)); + } + } + } + addSubstrings(context, String.valueOf(EOS)); + } catch (FileNotFoundException ex) { + Messages.severe(NgramModel.class.getName() + ": " + ex); + } catch (IOException ex) { + Messages.severe(NgramModel.class.getName() + ": " + ex); + } + } + + /** + * Add all substrings in this string to the NgramModel + * + * @param context the preceding context, possibly empty + * @param text the non-empty input string + */ + protected void addSubstrings(String context, String text) { + if (text.length() < 1) { + throw new IllegalArgumentException("Cannot extract n-grams from empty text"); + } + String s = context + text; + // extract all substrings + for (int high = context.length() + 1; high <= s.length(); ++high) { + for (int low = Math.max(0, high - order); low < high; ++low) { + addEntry(s.substring(low, high)); + } + } + // the normalization of 1-grams + addEntries("", text.length()); + } + + /** + * Compute the log-likelihood (per character) of the text contained in a + * file + * + * @param is the InputStream containing the text + * @param contextLength the length of the context for the evaluation of the + * character probability + * @return + */ + public double logLikelihood(InputStream is, int contextLength) { + int nchar = 0; + double loglike = 0; + try { + BufferedReader reader + = new BufferedReader(new InputStreamReader(is)); + String context = String.valueOf(BOS); + + while (reader.ready()) { + String line = StringNormalizer.reduceWS(reader.readLine()); + if (!line.isEmpty()) { + String input + = (context.charAt(0) == BOS) + ? line + : " " + line; + + nchar += input.length(); + loglike += logLikelihood(context, input); + if (input.length() > contextLength) { + context = input.substring(input.length() - contextLength); + } else { + String s = context + input; + context = s.substring(Math.max(0, s.length() - contextLength)); + } + } + loglike += logLikelihood(context, String.valueOf(EOS)); + ++nchar; + } + } catch (IOException ex) { + Messages.warning(NgramModel.class.getName() + ": " + ex.getMessage()); + } + + return loglike / nchar; + } + + /** + * + * @param context the preceding context + * @param input a non-empty string + * @return the log probability of the string after the given context + */ + public double logLikelihood(String context, String input) { + int contextLength = context.length(); + double loglike = 0; + for (int pos = 0; pos < input.length(); ++pos) { + String s; + if (pos >= contextLength) { + s = input.substring(pos - contextLength, pos + 1); + } else { + s = context.substring(pos) + + input.substring(0, pos + 1); + } + loglike += Math.log(smoothProb(s)); + } + return loglike; + } + + /** + * Compute probability of a character after a given context. This + * implementation only takes into account the preceding context + * + * @param context a sequence of characters + * @param c a character + * @return the log-probability (base e) that the character c follows the + * given context + */ + public double logProb(String context, char c) { + double res = 0; + int len = context.length() + 1; // the length of the context + character + String s = (len > order) + ? context.substring(len - order) + c + : context + c; + double p = smoothProb(s); + + return (p > 0) + ? Math.log(p) + : Double.NEGATIVE_INFINITY; + } + + /** + * + * @return string representation of the NgramModel: keys and values + */ + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + for (String key : occur.keySet()) { + String s = key.replaceAll(String.valueOf(BOS), "") + .replaceAll(String.valueOf(EOS), ""); + builder.append("'").append(s).append("' ") + .append(occur.get(key)).append('\n'); + } + return builder.toString(); + } + + /** + * Show differences between two NgramModels (debug function) + * + * @param other another NgramModel (order must coincide) + */ + public void showDiff(NgramModel other) { + if (this.order != other.order) { + throw new IllegalArgumentException("Illegal comparison " + + "of n-gram models with different n"); + } + for (String s : this.occur.keySet()) { + if (s.length() > 0) { + int val1 = this.occur.get(s).getValue(); + int val2 = other.occur.containsKey(s) + ? other.occur.get(s).getValue() : 0; + if (val1 != val2) { + System.out.println(s.replaceAll(String.valueOf(BOS), "") + .replaceAll(String.valueOf(EOS), "") + + " " + val1 + " " + val2); + } + } + } + for (String s : other.occur.keySet()) { + if (s.length() > 0 && !this.occur.containsKey(s)) { + int val2 = other.occur.get(s).getValue(); + System.out.println(s.replaceAll(String.valueOf(BOS), "") + .replaceAll(String.valueOf(EOS), "") + + " 0 " + val2); + } + } + } + + /** + * Compare two NgramModels + * + * @param other + * @return true if they store the same content + */ + public boolean equals(NgramModel other) { + if (this.order != other.order) { + return false; + } else { + for (String s : this.occur.keySet()) { + int val1 = this.occur.get(s).getValue(); + int val2 = other.occur.containsKey(s) + ? other.occur.get(s).getValue() : 0; + if (val1 != val2) { + return false; + } + + } + for (String s : other.occur.keySet()) { + if (!this.occur.containsKey(s)) { + int val2 = other.occur.get(s).getValue(); + if (val2 != 0) { + return false; + } + } + } + } + return true; + } + + @Override + public boolean equals(Object o) { + if (o instanceof NgramModel) { + NgramModel other = (NgramModel) o; + return equals(other); + } else { + return false; + } + } + + @Override + public int hashCode() { + return occur.hashCode(); + } + + /** + * Main function. + * + * @param args + * @throws java.io.FileNotFoundException + */ + public static void main(String[] args) throws FileNotFoundException { + NgramModel ngram = null; + File fout = null; + int order = 0; + + if (args.length == 0) { + System.err.println("Usage: Ngram [-n NgramModelOrder]" + + " [-i InputNgramModelFile | -o OutputNgramFile]" + + " file1 file2 ...."); + } else { + for (int k = 0; k < args.length; ++k) { + String arg = args[k]; + + if (arg.equals("-n")) { + order = Integer.parseInt(args[++k]); + ngram = new NgramModel(order); + } else if (arg.equals("-i")) { + File fin = new File(args[++k]); + ngram = new NgramModel(fin); + } else if (arg.equals("-o")) { + fout = new File(args[++k]); + } else if (ngram != null) { + File file = new File(arg); + InputStream is = new FileInputStream(file); + if (fout != null) { + ngram.addText(is); + } else if (order > 1) { + double res = ngram.logLikelihood(is, order - 1); + System.out.println(res); + } + } + } + if (fout != null) { + ngram.save(fout); + } + } + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/ngram/NgramModelExaminer.java b/ocrevalUAtion/src/main/java/eu/digitisation/ngram/NgramModelExaminer.java new file mode 100644 index 00000000..e592cd29 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/ngram/NgramModelExaminer.java @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +package eu.digitisation.ngram; + +import java.io.File; +import java.io.FileNotFoundException; + +/** + * + * @author R.C.C. + */ +public class NgramModelExaminer { + public static void main(String[] args) throws FileNotFoundException { + NgramModel model = new NgramModel(new File(args[0])); + + + for (int n = 1; n < args.length; ++n) { + String key = args[n]; + String head = key.substring(0, key.length() - 1); + int times = model.occurrences(key); + int total = model.occurrences(head); + System.out.println(times + " / " + total); + } + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/ngram/NgramPerplexityEvaluator.java b/ocrevalUAtion/src/main/java/eu/digitisation/ngram/NgramPerplexityEvaluator.java new file mode 100644 index 00000000..a4740ad0 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/ngram/NgramPerplexityEvaluator.java @@ -0,0 +1,63 @@ +package eu.digitisation.ngram; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.InputStream; + +/** + * Perplexity evaluator based on an n-gram model + * + */ +public class NgramPerplexityEvaluator implements PerplexityEvaluator { + + NgramModel ngram; + + public NgramPerplexityEvaluator(NgramModel ngram) { + this.ngram = ngram; + } + + public NgramPerplexityEvaluator(File file) { + ngram = new NgramModel(file); + } + + /** + * Calculates perplexity for each character of a given text. + * + * @param textToEvaluate perplexity of characters contained in this text is + * calculated + * @param contextLength the length of character context that is considered + * when calculating perplexity + * @return array of perplexity values, each item in the array is a + * perplexity of corresponding character in the given text. + */ + @Override + public double[] calculatePerplexity(String textToEvaluate, int contextLength) { + int textLen = textToEvaluate.length(); + double[] logprobs = new double[textLen]; + for (int pos = 0; pos < textLen; ++pos) { + int beg = Math.max(0, pos - contextLength); + String context = textToEvaluate.substring(beg, pos); + logprobs[pos] = ngram.logProb(context, textToEvaluate.charAt(pos)); + } + return logprobs; + } + + public static void main(String[] args) throws FileNotFoundException { + NgramModel model = new NgramModel(new File(args[0])); + int contextLenght = Integer.parseInt(args[1]); + InputStream is = (args.length == 3) + ? new FileInputStream(new File(args[2])) + : System.in; + + TextPerplexity result + = new TextPerplexity(model, is, contextLenght); + + String text = result.getText(); + double[] perps = result.getPerplexities(); + + for (int n = 0; n < text.length(); ++n) { + System.out.println(text.charAt(n) + " " + perps[n]); + } + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/ngram/PerplexityEvaluator.java b/ocrevalUAtion/src/main/java/eu/digitisation/ngram/PerplexityEvaluator.java new file mode 100644 index 00000000..e33574ea --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/ngram/PerplexityEvaluator.java @@ -0,0 +1,17 @@ +package eu.digitisation.ngram; + +/** + * Interface for perplexity evaluator. It calculates perplexity of characters in a given text. + * @author tparkola + * + */ +public interface PerplexityEvaluator { + + /** + * Calculates perplexity for each character of a given text. + * @param textToEvaluate perplexity of characters contained in this text is calculated + * @param contextLength the length of character context that is considered when calculating perplexity + * @return array of perplexity values, each item in the array is a perplexity of corresponding character in the given text. + */ + public double[] calculatePerplexity(String textToEvaluate, int contextLength); +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/ngram/PerplexityEvaluatorAssesmentHelper.java b/ocrevalUAtion/src/main/java/eu/digitisation/ngram/PerplexityEvaluatorAssesmentHelper.java new file mode 100644 index 00000000..c5132a99 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/ngram/PerplexityEvaluatorAssesmentHelper.java @@ -0,0 +1,85 @@ +package eu.digitisation.ngram; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.IOException; +import java.io.OutputStreamWriter; + +public class PerplexityEvaluatorAssesmentHelper { + public static void main(String[] args) throws IOException { + // langModel, OCRFile, contextLengthRange, resultFile + File langModelFile = new File(args[0]); + File OCRFile = new File(args[1]); + File outputFile = new File(args[3]); + + String OCRText = extractString(OCRFile); + + NgramModel providedModel = new NgramModel(langModelFile); + + ContextLengthRange contextLengthRange = ContextLengthRange + .parseContextLengthRange(args[2]); + + double[][] perplexities = new double[contextLengthRange.getEnd() + - contextLengthRange.getStart() + 1][OCRText.length()]; + + PerplexityEvaluator logPerplexityEvaluator = new NgramPerplexityEvaluator( + providedModel); + + for (int i = contextLengthRange.getStart(); i <= contextLengthRange + .getEnd(); i++) { + perplexities[i - contextLengthRange.getStart()] = logPerplexityEvaluator.calculatePerplexity( + OCRText, i); + } + + BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outputFile))); + + bw.write(providedModel.order + "-gram model results:"); + bw.newLine(); + printPerplexities(contextLengthRange, OCRText, perplexities, bw); + } + + private static void printPerplexities( + ContextLengthRange contextLengthRange, String OCRText, + double[][] perplexities, BufferedWriter bw) throws IOException { +// System.out.print("Letter\t"); + bw.write("Letter\t"); + for (int i = contextLengthRange.getStart(); i <= contextLengthRange + .getEnd(); i++) { +// System.out.print("PC" + i + "\t"); + bw.write("PC" + i + "\t"); + } +// System.out.println(); + bw.newLine(); + for (int j = 0; j < OCRText.length(); j++) { +// System.out.print(OCRText.charAt(j) + "\t"); + bw.write(OCRText.charAt(j) + "\t"); + for (int i = 0; i < perplexities.length; i++) { +// System.out.print(perplexities[i][j] + "\t"); + bw.write(perplexities[i][j] + "\t"); + } +// System.out.println(); + bw.newLine(); + } + bw.close(); + } + + private static String extractString(File OCRFile) + throws FileNotFoundException, IOException { + BufferedReader reader = new BufferedReader(new FileReader(OCRFile)); + + StringBuffer OCRFileText = new StringBuffer(); + + String line = null; + while ((line = reader.readLine()) != null) { + OCRFileText.append(line + "\n"); + } + + reader.close(); + String OCRText = OCRFileText.toString(); + return OCRText; + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/ngram/TextPerplexity.java b/ocrevalUAtion/src/main/java/eu/digitisation/ngram/TextPerplexity.java new file mode 100644 index 00000000..acd6886e --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/ngram/TextPerplexity.java @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.ngram; + +import eu.digitisation.log.Messages; +import static eu.digitisation.ngram.NgramModel.BOS; +import eu.digitisation.text.StringNormalizer; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; + +/** + * + * @author R.C.C. + */ +public class TextPerplexity { + + StringBuilder text; + List perplexities; + + public TextPerplexity(NgramModel ngram, InputStream is, int contextLength) { + text = new StringBuilder(); + perplexities = new ArrayList(); + try { + BufferedReader reader + = new BufferedReader(new InputStreamReader(is)); + String context = String.valueOf(BOS); + + while (reader.ready()) { + String line = StringNormalizer.reduceWS(reader.readLine()); + if (!line.isEmpty()) { + String input + = (context.charAt(0) == BOS) + ? line + : " " + line; + + for (int pos = 0; pos < input.length(); ++pos) { + String s; + if (pos >= contextLength) { + s = input.substring(pos - contextLength, pos + 1); + } else { + s = context.substring(pos) + + input.substring(0, pos + 1); + } + text.append(input.charAt(pos)); + perplexities.add(Math.log(ngram.smoothProb(s))); + + if (input.length() > contextLength) { + context = input.substring(input.length() - contextLength); + } else { + s = context + input; + context = s.substring(Math.max(0, s.length() - contextLength)); + } + } + + } + } + } catch (IOException ex) { + Messages.warning(NgramModel.class.getName() + ": " + ex.getMessage()); + } + + } + + public String getText() { + return text.toString(); + } + + public double[] getPerplexities() { + double[] array = new double[perplexities.size()]; + for (int n = 0; n < array.length; ++n) { + array[n] = perplexities.get(n); + } + return array; + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/output/Browser.java b/ocrevalUAtion/src/main/java/eu/digitisation/output/Browser.java new file mode 100644 index 00000000..79509773 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/output/Browser.java @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.output; + +import eu.digitisation.log.Messages; +import java.awt.Desktop; +import java.awt.Desktop.Action; +import java.io.IOException; +import java.net.URI; + +/** + * Open a file or URL with an operating system application + * + * @author R.C.C. + */ +public class Browser { + + /** + * Open a URI + * + * @param uri the location of the file or resource + */ + public static void open(URI uri) { + System.out.println(uri); + if (Desktop.isDesktopSupported()) { + Desktop desktop = Desktop.getDesktop(); + if (desktop.isSupported(Action.BROWSE)) { + try { + Desktop.getDesktop().browse(uri); + } catch (IOException ex) { + Messages.info(Browser.class.getName() + ": " + ex); + } + } + } else { + try { + Runtime.getRuntime().exec( + "rundll32 url.dll,FileProtocolHandler " + uri); + } catch (IOException ex) { + Messages.info(Browser.class.getName() + ": " + ex); + } + } + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/output/CharStatTable.java b/ocrevalUAtion/src/main/java/eu/digitisation/output/CharStatTable.java new file mode 100644 index 00000000..8b10e5f2 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/output/CharStatTable.java @@ -0,0 +1,223 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.output; + +import eu.digitisation.distance.EdOp; +import eu.digitisation.distance.StringEditDistance; +import eu.digitisation.math.BiCounter; +import eu.digitisation.xml.DocumentBuilder; +import org.w3c.dom.Element; + +/** + * Provide statistics of the differences between two texts + * + * @author R.C.C. + */ +public class CharStatTable extends BiCounter { + private static final long serialVersionUID = 1L; + + /** + * Create empty CharStatTable + */ + public CharStatTable() { + super(); + } + + /** + * Separate statistics of errors for every character + * + * @param s1 + * the reference text + * @param s2 + * the fuzzy text + */ + public CharStatTable(String s1, String s2) { + super(); + add(StringEditDistance.operations(s1, s2)); + } + + /** + * Separate statistics of errors for every character form a collection of + * texts + * + * @param array1 + * an array of reference texts + * @param array2 + * an array of fuzzy texts + */ + public CharStatTable(String[] array1, String[] array2) { + if (array1.length == array2.length) { + for (int n = 0; n < array1.length; ++n) { + add(StringEditDistance.operations(array1[n], array2[n])); + } + } else { + throw new java.lang.IllegalArgumentException( + "Arrays of different length"); + } + } + + /** + * Add statistic for a pair of strings + * + * @param s1 + * the reference text + * @param s2 + * the fuzzy text + */ + public void add(String s1, String s2) { + add(StringEditDistance.operations(s1, s2)); + } + + /** + * Separate statistics of errors for every character + * + * @return an element containing table with the statistics: one character + * per row and one edit operation per column. + */ + public Element asTable() { + DocumentBuilder builder = new DocumentBuilder("table"); + Element table = builder.root(); + Element row = builder.addElement("tr"); + + // features + table.setAttribute("border", "1"); + // header + builder.addTextElement(row, "th", "Character"); + builder.addTextElement(row, "th", "Character name"); + builder.addTextElement(row, "th", "Hex code"); + builder.addTextElement(row, "th", "Total"); + builder.addTextElement(row, "th", "Keep"); + builder.addTextElement(row, "th", "Insert"); + builder.addTextElement(row, "th", "Substitute"); + builder.addTextElement(row, "th", "Delete"); + builder.addTextElement(row, "th", "Error rate (%)"); + builder.addTextElement(row, "th", "Accuracy (%)"); + + // content + for (Character c : leftKeySet()) { + int spu = value(c, EdOp.INSERT); + int sub = value(c, EdOp.SUBSTITUTE); + int add = value(c, EdOp.DELETE); + int kep = value(c, EdOp.KEEP); + int tot = kep + sub + add; + double rate = (spu + sub + add) / (double) tot * 100; + double accuracy = (tot - (spu + sub + add)) / (double) tot * 100; + row = builder.addElement("tr"); + builder.addTextElement(row, "td", c.toString()); + builder.addTextElement(row, "td", Character.getName((int) c)); + builder.addTextElement(row, "td", Integer.toHexString(c)); + builder.addTextElement(row, "td", String.valueOf(tot)); + builder.addTextElement(row, "td", String.valueOf(kep)); + builder.addTextElement(row, "td", String.valueOf(spu)); + builder.addTextElement(row, "td", String.valueOf(sub)); + builder.addTextElement(row, "td", String.valueOf(add)); + builder.addTextElement(row, "td", String.format("%.2f", rate)); + builder.addTextElement(row, "td", String.format("%.2f", accuracy)); + } + return builder.document().getDocumentElement(); + } + + /** + * Prints separate statistics of errors for every character + * + * @param recordSeparator + * text between data records + * @param fieldSeparator + * text between data fields + * @return text with the statistics: every character separated by a record + * separator and every type of edit operation separated by field + * separator. + * + */ + public StringBuilder asCSV(String recordSeparator, String fieldSeparator) { + StringBuilder builder = new StringBuilder(); + + builder.append("Character") + .append(fieldSeparator).append("Hex code") + .append(fieldSeparator).append("Total") + .append(fieldSeparator).append("Keep") + .append(fieldSeparator).append("Insert") + .append(fieldSeparator).append("Substitute") + .append(fieldSeparator).append("Delete") + .append(fieldSeparator).append("Error rate (%)") + .append(fieldSeparator).append("Accuracy (%)"); + + for (Character c : leftKeySet()) { + int spu = value(c, EdOp.INSERT); + int sub = value(c, EdOp.SUBSTITUTE); + int add = value(c, EdOp.DELETE); + int kep = value(c, EdOp.KEEP); + int tot = kep + sub + add; + double rate = (spu + sub + add) / (double) tot * 100; + double accuracy = (tot - (spu + sub + add)) / (double) tot * 100; + builder.append(recordSeparator); + builder.append(c) + .append(fieldSeparator).append(Integer.toHexString(c)) + .append(fieldSeparator).append(tot) + .append(fieldSeparator).append(kep) + .append(fieldSeparator).append(spu) + .append(fieldSeparator).append(sub) + .append(fieldSeparator).append(add) + .append(fieldSeparator).append(String.format("%.2f", rate)) + .append(fieldSeparator).append( + String.format("%.2f", accuracy)); + } + return builder; + } + + /** + * Extract CER from character statistics + * + * @return the global CER + */ + public double cer() { + int spu = 0; + int sub = 0; + int add = 0; + int tot = 0; + + for (Character c : leftKeySet()) { + spu += value(c, EdOp.INSERT); + sub += value(c, EdOp.SUBSTITUTE); + add += value(c, EdOp.DELETE); + tot += value(c, EdOp.KEEP) + + value(c, EdOp.SUBSTITUTE) + + value(c, EdOp.DELETE); + } + + return (spu + sub + add) / (double) tot; + } + + public double accuracy() { + int spu = 0; + int sub = 0; + int add = 0; + int tot = 0; + + for (Character c : leftKeySet()) { + spu += value(c, EdOp.INSERT); + sub += value(c, EdOp.SUBSTITUTE); + add += value(c, EdOp.DELETE); + tot += value(c, EdOp.KEEP) + + value(c, EdOp.SUBSTITUTE) + + value(c, EdOp.DELETE); + } + + return (tot - (spu + sub + add)) / (double) tot; + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/output/ErrorMeasure.java b/ocrevalUAtion/src/main/java/eu/digitisation/output/ErrorMeasure.java new file mode 100644 index 00000000..b0dde016 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/output/ErrorMeasure.java @@ -0,0 +1,148 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.output; + +import eu.digitisation.distance.ArrayEditDistance; +import eu.digitisation.distance.EditDistanceType; +import eu.digitisation.distance.StringEditDistance; +import eu.digitisation.document.TermFrequencyVector; +import eu.digitisation.document.TokenArray; +import eu.digitisation.log.Messages; +import eu.digitisation.math.MinimalPerfectHash; + +/** + * Computes character and word error rates by comparing two texts + * + * @version 2012.06.20 + */ +public class ErrorMeasure { + + /** + * Compute character error rate using Levenshtein distance + * + * @param s1 the reference text + * @param s2 fuzzy text + * @return character error rate with respect to the reference file + */ + public static double cer(String s1, String s2) { + int l1 = s1.length(); + int l2 = s2.length(); + double delta = (100.00 * Math.abs(l1 - l2)) / (l1 + l2); + + if (delta > 20) { + Messages.warning("Files differ a " + + String.format("%.2f", delta) + " % in character length"); + } + + return StringEditDistance.distance(s1, s2, EditDistanceType.LEVENSHTEIN) + / (double) l1; + } + + /** + * Compute character error rate using Damerau-Levenshtein distance + * + * @param s1 the reference text + * @param s2 fuzzy text + * @return character error rate with respect to the reference file + */ + public static double cerDL(String s1, String s2) { + int l1 = s1.length(); + int l2 = s2.length(); + double delta = (100.00 * Math.abs(l1 - l2)) / (l1 + l2); + + if (delta > 20) { + Messages.warning("Files differ a " + + String.format("%.2f", delta) + " % in character length"); + } + + return StringEditDistance.distance(s1, s2, EditDistanceType.DAMERAU_LEVENSHTEIN) + / (double) l1; + } + + /** + * Compute word error rate + * + * @param a1 array of integers + * @param a2 array of integers + * @return error rate + */ + private static double wer(TokenArray a1, TokenArray a2) { + int l1 = a1.length(); + int l2 = a2.length(); + double delta = (100.00 * Math.abs(l1 - l2)) / (l1 + l2); + + if (delta > 20) { + Messages.warning("Files differ a " + + String.format("%.2f", delta) + " % in word length"); + } + + return ArrayEditDistance.distance(a1.tokens(), a2.tokens(), + EditDistanceType.LEVENSHTEIN) / (double) l1; + } + + /** + * Compute word recall rate + * + * @param a1 first TokenArray + * @param a2 second TokenArray + * @return word recall (fraction of words in a1 also in a2) + */ + public static double wordRecall(TokenArray a1, TokenArray a2) { + int l1 = a1.length(); + int l2 = a2.length(); + double delta = (100.00 * Math.abs(l1 - l2)) / (l1 + l2); + + if (delta > 20) { + Messages.warning("Files differ a " + + String.format("%.2f", delta) + " % in word length"); + } + + int indel = ArrayEditDistance.distance(a1.tokens(), a2.tokens(), + EditDistanceType.INDEL); + return (l1 + l2 - indel) / (double) (2 * l1); + } + + /** + * Compute word error rate + * + * @param s1 reference text + * @param s2 fuzzy text + * @return word error rate with respect to first file + */ + public static double wer(String s1, String s2) { + MinimalPerfectHash mph = new MinimalPerfectHash(false); // case unsensitive + TokenArray a1 = new TokenArray(mph, s1); + TokenArray a2 = new TokenArray(mph, s2); + + return wer(a1, a2); + } + + /** + * Compute bag-of-word error rate + * + * @param s1 reference string + * @param s2 fuzzy string string + * @return the word error rate between the (unsorted) strings + */ + public static double ber(String s1, String s2) { + TermFrequencyVector tf1 = new TermFrequencyVector(s1); + TermFrequencyVector tf2 = new TermFrequencyVector(s2); + + return tf1.distance(tf2) / (double) tf1.total(); + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/output/OutputFileSelector.java b/ocrevalUAtion/src/main/java/eu/digitisation/output/OutputFileSelector.java new file mode 100644 index 00000000..ad8ae70b --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/output/OutputFileSelector.java @@ -0,0 +1,76 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.output; + +import java.io.File; +import javax.swing.JFileChooser; +import javax.swing.JFrame; +import javax.swing.JOptionPane; + +/** + * File chooser with confirmation dialog to avoid accidental overwrite + * + * @author R.C.C. + */ +public class OutputFileSelector extends JFileChooser { + + private static final long serialVersionUID = 1L; + private static File dir; // default directory + + /** + * Default constructor + */ + public OutputFileSelector() { + super(); + } + + /** + * + * @param dir the default directory + * @param file the preselected selection + * @return the selected selection + */ + public File choose(File dir, File file) { + // Use last choice + if (OutputFileSelector.dir == null) { + OutputFileSelector.dir = dir; + } + setCurrentDirectory(OutputFileSelector.dir); + setSelectedFile(file); + + int returnVal = showOpenDialog(OutputFileSelector.this); + + if (returnVal == JFileChooser.APPROVE_OPTION) { + File selection = getSelectedFile(); + OutputFileSelector.dir = selection.getParentFile(); + + if (selection != null && selection.exists()) { + int response = JOptionPane.showConfirmDialog(new JFrame().getContentPane(), + "The file " + selection.getName() + + " already exists.\n" + + "Do you want to replace the existing file?", + "Overwrite file", JOptionPane.YES_NO_OPTION, + JOptionPane.WARNING_MESSAGE); + + return (response == JOptionPane.YES_NO_OPTION) ? selection : null; + } + return selection; + } + return null; + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/output/Report.java b/ocrevalUAtion/src/main/java/eu/digitisation/output/Report.java new file mode 100644 index 00000000..3dc07ca7 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/output/Report.java @@ -0,0 +1,190 @@ +/* + * Copyright (C) 2013 IMPACT Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.output; + +import eu.digitisation.distance.Aligner; +import eu.digitisation.distance.EdOpWeight; +import eu.digitisation.distance.EditDistance; +import eu.digitisation.distance.EditSequence; +import eu.digitisation.distance.OcrOpWeight; +import eu.digitisation.document.TermFrequencyVector; +import eu.digitisation.input.Batch; +import eu.digitisation.input.Parameters; +import eu.digitisation.input.SchemaLocationException; +import eu.digitisation.input.WarningException; +import eu.digitisation.log.Messages; +import eu.digitisation.math.Pair; +import eu.digitisation.text.CharFilter; +import eu.digitisation.text.StringNormalizer; +import eu.digitisation.text.Text; +import eu.digitisation.text.WordSet; +import eu.digitisation.xml.DocumentBuilder; + +import java.io.File; +import java.nio.file.Paths; + +import org.w3c.dom.Element; + +/** + * Create a report in HTML format + * + * @author R.C.C + */ +public class Report extends DocumentBuilder { + + Element head; + Element body; + private CharStatTable stats; + + /** + * Initial settings: create an empty HTML document + */ + private void init() { + head = addElement("head"); + body = addElement("body"); + // metadata + Element meta = addElement(head, "meta"); + meta.setAttribute("http-equiv", "content-type"); + meta.setAttribute("content", "text/html; charset=UTF-8"); + } + + /** + * Insert a table at the end of the document body + * + * @param content + * the table content + * @return the table element + */ + private Element addTable(Element parent, String[][] content) { + Element table = addElement(parent, "table"); + table.setAttribute("border", "1"); + for (String[] row : content) { + Element tabrow = addElement(table, "tr"); + for (String cell : row) { + addTextElement(tabrow, "td", cell); + } + } + return table; + } + + /** + * + * @param batch + * a batch of file pairs + * @param pars + * input parameters + * @throws eu.digitisation.input.WarningException + * @throws eu.digitisation.input.SchemaLocationException + */ + public Report(Batch batch, Parameters pars) + throws WarningException, SchemaLocationException { + super("html"); + init(); + + File swfile = pars.swfile.getValue(); + EdOpWeight w = new OcrOpWeight(pars); + stats = new CharStatTable(); + CharFilter filter; + + // optional eqfile + if (pars.compatibility.getValue() != null) { + filter = new CharFilter(pars.compatibility.getValue(), + pars.eqfile.getValue()); + } else { + filter = new CharFilter(pars.compatibility.getValue()); + } + + Element summaryTab; + int numwords = 0; // number of words in GT + int wdist = 0; // word distances + int bdist = 0; // bag-of-words distanbces + + addTextElement(body, "h2", "General results"); + summaryTab = addElement(body, "div"); + addTextElement(body, "h2", "Difference spotting"); + + for (int n = 0; n < batch.size(); ++n) { + Pair input = batch.pair(n); + Messages.info("Processing " + input.first.getName()); + Text gt = new Text(input.first); + Text ocr = new Text(input.second); + String gtref = pars.ignoreDiacritics.getValue() // remove spurious + // marks + ? gt.toString(filter).replaceAll( + " \\p{InCombiningDiacriticalMarks}+", " ") + : gt.toString(filter); + String ocrref = pars.ignoreDiacritics.getValue() + ? ocr.toString(filter) // remove spurious marks + .replaceAll(" \\p{InCombiningDiacriticalMarks}+", " ") + : ocr.toString(filter); + String gts = StringNormalizer.canonical(gtref, + pars.ignoreCase.getValue(), + pars.ignoreDiacritics.getValue(), + false); + String ocrs = StringNormalizer.canonical(ocrref, + pars.ignoreCase.getValue(), + pars.ignoreDiacritics.getValue(), + false); + EditSequence eds = new EditSequence(gts, ocrs, w, 2000); + TermFrequencyVector gtv = new TermFrequencyVector(gts); + TermFrequencyVector ocrv = new TermFrequencyVector(ocrs); + Element alitab = Aligner.bitext(input.first.getName(), + input.second.getName(), gtref, ocrref, w, eds); + int[] wd = (swfile == null) + ? EditDistance.wordDistance(gts, ocrs, 1000) + : EditDistance.wordDistance(gts, ocrs, new WordSet(swfile), + 1000); + + stats.add(eds.stats(gtref, ocrref, w)); + addTextElement(body, "div", " "); + addElement(body, alitab); + numwords += wd[0]; // length (words) in gts + wdist += wd[2]; // word-based distance + bdist += gtv.distance(ocrv); + } + // Summary table + double cer = stats.cer(); + double accuracy = stats.accuracy(); + double wer = wdist / (double) numwords; + double ber = bdist / (double) numwords; + String[][] summaryContent = { + { "CER %", String.format("%.2f", cer * 100) }, + // {"CER (with swaps)", String.format("%.2f", cerDL * 100)}, + { "WER %", String.format("%.2f", wer * 100) }, + { "WER % (order independent)", String.format("%.2f", ber * 100) }, + { "Character accuracy %", String.format("%.2f", accuracy * 100) } + }; + addTable(summaryTab, summaryContent); + // CharStatTable + addTextElement(body, "h2", "Error rate per character and type"); + addElement(body, stats.asTable()); + } + + public CharStatTable getStats() { + return stats; + } + + public static void main(String[] args) { + if (args.length != 2) { + System.err.println("Usage: aligner file1 file2"); + } else { + Element alitab = + Aligner.alignmentMap("", "", args[0], args[1], null); + } + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/text/CharFilter.java b/ocrevalUAtion/src/main/java/eu/digitisation/text/CharFilter.java new file mode 100644 index 00000000..5d3f7037 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/text/CharFilter.java @@ -0,0 +1,271 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.text; + +import eu.digitisation.input.ExtensionFilter; +import eu.digitisation.log.Messages; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.util.HashMap; +import java.util.Map; + +/** + * Transform text according to a mapping between (source, target) Unicode + * character sequences. This can be useful, for example, to replace Unicode + * characters which are not supported by the browser or editor with printable + * ones. It also performs canonicalization an returns the recommended normal + * form (NFC = composed or NFKC if compatibility mode is selected). + * + * @version 2012.06.20 + */ +public class CharFilter extends HashMap { + + private static final long serialVersionUID = 1L; + boolean compatibility; // Unicode compatibility mode + + /** + * Default constructor + */ + public CharFilter() { + super(); + this.compatibility = false; + } + + /** + * Default constructor + * + * @param compatibility + * the Unicode compatibility mode (true means activated) + * @param file + * a CSV file with one transformation per line, each line + * contains two Unicode hex sequences (and comments) separated + * with commas + */ + public CharFilter(boolean compatibility, File file) { + super(); + this.compatibility = compatibility; + addFilter(file); + } + + /** + * Default constructor + * + * @param compatibility + * the Unicode compatibility mode (true means activated) + */ + public CharFilter(boolean compatibility) { + super(); + this.compatibility = compatibility; + } + + /** + * Constructor that inherits all entries of the given source map. + * + * @param compatibility + * @param source + */ + public CharFilter(boolean compatibility, Map source) { + super(source.size()); + this.compatibility = compatibility; + this.putAll(source); + } + + /** + * Load the transformation map from a CSV file: one transformation per line, + * each line contains two Unicode hex sequences (and comments) separated + * with commas + * + * @param file + * the CSV file (or directory with CSV files) with the equivalent + * sequences + */ + public CharFilter(File file) { + this.compatibility = false; + addFilter(file); + } + + /** + * Add files to filter + * + * @param file + * the CSV file (or directory with CSV files) with the equivalent + * sequences + */ + public final void addFilter(File file) { + if (file.isDirectory()) { + String[] filenames = file.list(new ExtensionFilter(".csv")); + for (String filename : filenames) { + addCSV(new File(filename)); + } + } else if (file.isFile()) { + addCSV(file); + } + } + + /** + * Add the equivalences contained in a CSV file + * + * @param file + * the CSV file + */ + private void addCSV(File file) { + try { + BufferedReader reader = new BufferedReader(new FileReader(file)); + while (reader.ready()) { + String line = reader.readLine(); + String[] tokens = line.split("([,;\t])"); + if (tokens.length > 1) { // allow comments in line + String key = UnicodeReader.codepointsToString(tokens[0]); + String value = UnicodeReader.codepointsToString(tokens[1]); + put(key, value); + } else { + throw new IOException("Wrong line" + line + + " at file " + file); + } + } + reader.close(); + } catch (IOException ex) { + + } + } + + /** + * Add the equivalences in CSV format + * + * @param reader + * a BufferedReader with CSV lines + */ + public void addCSV(BufferedReader reader) { + try { + while (reader.ready()) { + String line = reader.readLine(); + String[] tokens = line.split("([,;\t])"); + if (tokens.length > 1) { // allow comments in line + String key = UnicodeReader.codepointsToString(tokens[0]); + String value = UnicodeReader.codepointsToString(tokens[1]); + put(key, value); + System.out.println(key + ", " + value); + } else { + throw new IOException("Wrong CSV line" + line); + } + } + reader.close(); + } catch (IOException ex) { + Messages.info(CharFilter.class.getName() + ": " + ex); + } + } + + /** + * Set the compatibility mode + * + * @param compatibility + * the compatibility mode + */ + public void setCompatibility(boolean compatibility) { + this.compatibility = compatibility; + } + + /** + * Find all occurrences of characters in a sequence and substitute them with + * the replacement specified by the transformation map. Remark: No + * replacement priority is guaranteed in case of overlapping matches. + * + * @param s + * the string to be transformed + * @return a new string with all the transformations performed + */ + public String translate(String s) { + String r = compatibility + ? StringNormalizer.compatible(s) + : StringNormalizer.composed(s); + for (Map.Entry entry : entrySet()) { + r = r.replaceAll(entry.getKey(), entry.getValue()); + } + return r; + } + + /** + * Converts the contents of a file into a CharSequence + * + * @param file + * the input file + * @return the file content as a CharSequence + */ + public CharSequence toCharSequence(File file) { + try { + FileInputStream input = new FileInputStream(file); + FileChannel channel = input.getChannel(); + java.nio.ByteBuffer buffer = channel.map( + FileChannel.MapMode.READ_ONLY, 0, channel.size()); + return java.nio.charset.Charset.forName("utf-8").newDecoder() + .decode(buffer); + } catch (IOException ex) { + Messages.info(CharFilter.class.getName() + ": " + ex); + } + return null; + } + + /** + * Translate all characters according to the transformation map + * + * @param infile + * the input file + * @param outfile + * the file where the output must be written + */ + public void translate(File infile, File outfile) { + try { + FileWriter writer = new FileWriter(outfile); + String input = toCharSequence(infile).toString(); + String output = translate(input); + + writer.write(output); + writer.flush(); + writer.close(); + } catch (IOException ex) { + Messages.info(CharFilter.class.getName() + ": " + ex); + } + } + + /** + * Translate (in place) all characters according to the transformation map + * + * @param file + * the input file + * + */ + public void translate(File file) { + try { + FileWriter writer = new FileWriter(file); + String input = toCharSequence(file).toString(); + String output = translate(input); + + writer.write(output); + writer.flush(); + writer.close(); + } catch (IOException ex) { + Messages.info(CharFilter.class.getName() + ": " + ex); + } + + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/text/CharMap.java b/ocrevalUAtion/src/main/java/eu/digitisation/text/CharMap.java new file mode 100644 index 00000000..fb4749cd --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/text/CharMap.java @@ -0,0 +1,199 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.text; + +import eu.digitisation.input.ExtensionFilter; +import eu.digitisation.log.Messages; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + + +/** + * Compares characters and strings according to a mapping between equivalent + * characters + * + * @author R.C.C. + */ +public class CharMap { + + /** + * Typical Options for character comparison + */ + public enum Option { + + /** + * True if case (uppercase or lower) matters + */ + CASE_AWARE, + /** + * True if punctuation marks, currency symbols and all other + * non-letter/non-number symbols matter + */ + PUNCTUATION_AWARE, + /** + * True if diacritics matter + */ + DIACRITICS_AWARE, + /** + * True if Unicode compatibility is active (e.g., between ligatures and + * non-ligatures) + */ + UNICODE_COMPATIBILITY + }; + + EnumMap options; // equivalence options + HashMap equivalences; // specific equivalences for Unicode characters + + /** + * Default constructor: all options true by default + */ + public CharMap() { + options = new EnumMap(Option.class); + for (Option option : Option.values()) { + options.put(option, Boolean.TRUE); + } + equivalences = new HashMap(); + } + + /** + * Constructor with selection of options + * + * @param ops the options whose value must be set to to true (all the other + * being false) + */ + public CharMap(Option[] ops) { + options = new EnumMap(Option.class); + for (Option option : Option.values()) { + options.put(option, Boolean.FALSE); + } + for (Option op : ops) { + options.put(op, Boolean.TRUE); + } + equivalences = new HashMap(); + } + + /** + * Set a value for an option + * + * @param option the option to be set + * @param value its value (true or false) + */ + public void setOption(Option option, boolean value) { + options.put(option, value); + } + + /** + * Read files containing equivalences between characters and sequences + * + * @param file the CSV file (or directory with CSV files) with the + * equivalences between chars and sequences + */ + public void addFilter(File file) { + if (file.isDirectory()) { + String[] filenames = file.list(new ExtensionFilter(".csv")); + for (String filename : filenames) { + addCSV(new File(filename)); + } + } else if (file.isFile()) { + addCSV(file); + } + } + + /** + * Add the equivalences contained in a CSV file + * + * @param file the CSV file + */ + private void addCSV(File file) { + try { + BufferedReader reader = new BufferedReader(new FileReader(file)); + while (reader.ready()) { + String line = reader.readLine(); + String[] tokens = line.split("([,;\t])"); + if (tokens.length > 1) { // allow comments in line + Character key = (char) Integer.parseInt(tokens[0].trim(), 16); + String value = UnicodeReader.codepointsToString(tokens[1]); + equivalences.put(key, value); + } else { + throw new IOException("Wrong line" + line + + " at file " + file); + } + } + reader.close(); + } catch (IOException ex) { + Messages.info(CharFilter.class.getName() + ": " + ex); + } + } + + /** + * Normalize characters in a string + * + * @param s a string of characters + * @return the normal form of s for string comparison + */ + public String normalForm(String s) { + String result = s; + System.out.println("S=" + s); + if (!options.get(Option.CASE_AWARE)) { + result = result.toLowerCase(Locale.getDefault()); + } + if (!options.get(Option.DIACRITICS_AWARE)) { + result = StringNormalizer.removeDiacritics(result); + } + if (!options.get(Option.PUNCTUATION_AWARE)) { + // keep only letters, diacritic marks an numbers + result = result.replaceAll("[^\\p{L}\\p{M}\\p{N}]", " "); + } + if (!options.get(Option.UNICODE_COMPATIBILITY)) { + result = StringNormalizer.compatible(result); + } + + for (Map.Entry entry : equivalences.entrySet()) { + result = result.replaceAll(String.valueOf(entry.getKey()), entry.getValue()); + } + + return StringNormalizer.reduceWS(result); + } + + /** + * Normalize a character + * + * @param c a character + * @return the normal form of c for character comparison + */ + public String normalForm(char c) { + return normalForm(String.valueOf(c)); + } + + /** + * Check if two characters are equivalent + * + * @param c1 the first character + * @param c2 the second character + * @return True if c1 is equivalent to c2 + */ + public boolean equiv(char c1, char c2) { + return normalForm(c1).equals(normalForm(c2)); + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/text/Encoding.java b/ocrevalUAtion/src/main/java/eu/digitisation/text/Encoding.java new file mode 100644 index 00000000..4c1db6ff --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/text/Encoding.java @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.text; + +import eu.digitisation.log.Messages; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.parser.txt.CharsetDetector; + +/** + * Detect the encoding of a text file + * + * @author R.C.C. + */ +public class Encoding { + + /** + * + * @param file a text file + * @return the encoding or Charset + */ + public static Charset detect(File file) { + try { + InputStream is = TikaInputStream.get(new FileInputStream(file)); + CharsetDetector detector = new CharsetDetector(); + detector.setText(is); + return Charset.forName(detector.detect().getName()); + } catch (IOException ex) { + Messages.info(Encoding.class.getName() + ": " + ex); + } + return null; + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/text/StringNormalizer.java b/ocrevalUAtion/src/main/java/eu/digitisation/text/StringNormalizer.java new file mode 100644 index 00000000..76133cdc --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/text/StringNormalizer.java @@ -0,0 +1,158 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.text; + +/** + * Normalizes strings: collapse whitespace and use composed form (see + * java.text.Normalizer.Form) + * + * @author R.C.C. + */ +public class StringNormalizer { + + final static java.text.Normalizer.Form decomposed = java.text.Normalizer.Form.NFD; + final static java.text.Normalizer.Form composed = java.text.Normalizer.Form.NFC; + static final java.text.Normalizer.Form compatible = java.text.Normalizer.Form.NFKC; + + /** + * Reduce whitespace (including line and paragraph separators) + * + * @param s + * a string. + * @return The string with simple spaces between words. + */ + public static String reduceWS(String s) { + return s.replaceAll("-\n", "").replaceAll( + "(\\p{Space}|\u2028|\u2029)+", " ").trim(); + } + + /** + * @param s + * a string + * @return the canonical representation of the string. + */ + public static String composed(String s) { + return java.text.Normalizer.normalize(s, composed); + } + + /** + * @param s + * a string + * @return the canonical representation of the string with normalized + * compatible characters. + */ + public static String compatible(String s) { + return java.text.Normalizer.normalize(s, compatible); + } + + /** + * @param s + * a string + * @return the string with all diacritics removed. + */ + public static String removeDiacritics(String s) { + return java.text.Normalizer.normalize(s, decomposed) + .replaceAll("\\p{InCombiningDiacriticalMarks}+", ""); + } + + /** + * @param s + * a string + * @return the string with all punctuation symbols removed. + */ + public static String removePunctuation(String s) { + return s.replaceAll("\\p{P}+", ""); + } + + /** + * @param s + * a string + * @return the string with leading and trailing whitespace and punctuation + * symbols removed. + */ + public static String trim(String s) { + return s.replaceAll("^(\\p{P}|\\p{Space})+", "") + .replaceAll("(\\p{P}|\\p{Space})+$", ""); + } + + /** + * + * @param s + * the input string + * @param ignoreCase + * true if case is irrelevant + * @param ignoreDiacritics + * true if diacritics are irrelevant + * @param ignorePunctuation + * true if punctuation is irrelevant + * @return the canonical representation for comparison + */ + public static String canonical(String s, + boolean ignoreCase, + boolean ignoreDiacritics, + boolean ignorePunctuation) { + + String res = (ignorePunctuation) ? removePunctuation(s) : s; + if (ignoreCase) { + if (ignoreDiacritics) { + return StringNormalizer.removeDiacritics(res).toLowerCase(); + } else { + return res.toLowerCase(); + } + } else if (ignoreDiacritics) { + return StringNormalizer.removeDiacritics(res); + } else { + return res; + } + } + + /** + * Remove everything except for letters (with diacritics), numbers and + * spaces + * + * @param s + * a string + * @return the string with only letters, numbers, spaces and diacritics. + */ + public static String strip(String s) { + return s.replaceAll("[^\\p{L}\\p{M}\\p{N}\\p{Space}]", ""); + } + + /** + * @param s + * a string + * @return the string with characters <, >, &, " escaped + */ + public static String encode(String s) { + StringBuilder result = new StringBuilder(); + for (Character c : s.toCharArray()) { + if (c.equals('<')) { + result.append("<"); + } else if (c.equals('>')) { + result.append(">"); + } else if (c.equals('"')) { + result.append("""); + } else if (c.equals('&')) { + result.append("&"); + } else { + result.append(c); + } + } + return result.toString(); + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/text/Text.java b/ocrevalUAtion/src/main/java/eu/digitisation/text/Text.java new file mode 100644 index 00000000..fc5ab2c2 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/text/Text.java @@ -0,0 +1,431 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.text; + +import eu.digitisation.input.FileType; +import eu.digitisation.input.SchemaLocationException; +import eu.digitisation.input.Settings; +import eu.digitisation.input.WarningException; +import eu.digitisation.layout.SortPageXML; +import eu.digitisation.log.Messages; +import eu.digitisation.xml.DocumentParser; +import eu.digitisation.xml.ElementList; +import eu.digitisation.xml.XPathFilter; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.Charset; +import java.util.List; +import java.util.Properties; +import javax.xml.xpath.XPathExpressionException; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +/** + * Extracts the text content in a file. Normalization collapses white-spaces and + * prefers composed form (see java.text.Normalizer.Form). For XML files, + * filtering options can be provided + * + * @author R.C.C. + */ +public class Text { + + StringBuilder builder; + static int maxlen; + Charset encoding; + XPathFilter filter; + + static { + Properties props = Settings.properties(); + maxlen = Integer.parseInt(props.getProperty("maxlen", "0").trim()); + Messages.info("max length of text set to " + maxlen); + try { + File inclusions = new File("inclusions.txt"); + File exclusions = new File("exclusions.txt"); + XPathFilter filter = inclusions.exists() + ? new XPathFilter(inclusions, exclusions) + : null; + } catch (IOException ex) { + Messages.info(Text.class.getName() + ": " + ex); + } catch (XPathExpressionException ex) { + Messages.info(Text.class.getName() + ": " + ex); + } + } + + /** + * Create TextContent from file + * + * @param file + * the input file + * @param encoding + * the text encoding for text files (optional; can be null) + * @param filter + * XPAthFilter for XML files (extracts textual content from + * selected elements) + * @throws eu.digitisation.input.WarningException + * @throws eu.digitisation.input.SchemaLocationException + */ + public Text(File file, Charset encoding, XPathFilter filter) + throws WarningException, SchemaLocationException { + + builder = new StringBuilder(); + this.encoding = encoding; + this.filter = filter; + + try { + FileType type = FileType.valueOf(file); + switch (type) { + case PAGE: + readPageFile(file); + break; + case TEXT: + readTextFile(file); + break; + case FR10: + readFR10File(file); + break; + case HOCR: + readHOCRFile(file); + break; + case ALTO: + readALTOFile(file); + break; + default: + throw new WarningException("Unsupported file format (" + + type + " format) for file " + + file.getName()); + } + } catch (eu.digitisation.input.SchemaLocationException ex) { + throw ex; + } catch (IOException ex) { + Messages.info(Text.class.getName() + ": " + ex); + } + builder.trimToSize(); + } + + /** + * Create Text from file + * + * @param file + * the input file + * @throws eu.digitisation.input.WarningException + * <<<<<<< HEAD + * @throws eu.digitisation.input.SchemaLocationException + * ======= >>>>>>> wip + */ + public Text(File file) + throws WarningException, SchemaLocationException { + this(file, null, null); + } + + /** + * Constructor only for debugging purposes + * + * @param s + * @throws eu.digitisation.input.WarningException + */ + public Text(String s) throws WarningException { + builder = new StringBuilder(); + encoding = Charset.forName("utf8"); + add(s); + } + + /** + * The length of the stored text + * + * @return the length of the stored text + */ + public int length() { + return builder.length(); + } + + /** + * The content as a string + * + * @return the text a String + */ + @Override + public String toString() { + return builder.toString(); + } + + /** + * The content as a string + * + * @param filter + * a CharFilter + * @return the text after the application of the filter + */ + public String toString(CharFilter filter) { + return filter == null + ? builder.toString() + : filter.translate(builder.toString()); + } + + /** + * Add content after normalization of whitespace and composition of + * diacritics + * + * @param s + * input text + */ + private void add(String s) throws WarningException { + String reduced = StringNormalizer.reduceWS(s); + if (reduced.length() > 0) { + String canonical = StringNormalizer.composed(reduced); + if (builder.length() > 0) { + builder.append(' '); + } + builder.append(canonical); + if (maxlen > 0 && builder.length() > maxlen) { + throw new WarningException("Text length limited to " + + maxlen + " characters"); + } + } + } + + private Document loadXMLFile(File file) { + Document doc = DocumentParser.parse(file); + String xmlEncoding = doc.getXmlEncoding(); + + if (xmlEncoding != null) { + encoding = Charset.forName(xmlEncoding); + Messages.info("XML file " + file.getName() + " encoding is " + + encoding); + } else { + if (encoding == null) { + encoding = Encoding.detect(file); + } + Messages.info("No encoding declaration in " + + file + ". Using " + encoding); + } + return doc; + } + + /** + * Read textual content and collapse whitespace: contiguous spaces are + * considered a single one + * + * @param file + * the input text file + */ + private void readTextFile(File file) throws WarningException { + // guess encoding if none is provided + if (encoding == null) { + encoding = Encoding.detect(file); + } + Messages.info("Text file " + file.getName() + " encoding is " + + encoding); + + // read content + try { + FileInputStream fis = new FileInputStream(file); + InputStreamReader isr = new InputStreamReader(fis, encoding); + BufferedReader reader = new BufferedReader(isr); + final StringBuilder completeText = new StringBuilder(); + + String line = null; + while ((line = reader.readLine()) != null) { + completeText.append(line.trim()); + completeText.append('\n'); + } + add(completeText.toString()); + reader.close(); + } catch (IOException ex) { + Messages.info(Text.class.getName() + ": " + ex); + } + } + + /** + * Reads textual content in a PAGE element of type TextRegion + * + * @param region + * the TextRegion element + */ + private void readPageTextRegion(Element region) throws IOException, + WarningException { + NodeList nodes = region.getChildNodes(); + for (int n = 0; n < nodes.getLength(); ++n) { + Node node = nodes.item(n); + if (node.getNodeName().equals("TextEquiv")) { + String text = node.getTextContent(); + add(text); + } + } + } + + /** + * Reads textual content in PAGE XML document. By default selects all + * TextREgion elements + * + * @param file + * the input XML file + */ + private void readPageFile(File file) throws IOException, WarningException { + Document doc = loadXMLFile(file); + Document sorted = SortPageXML.isSorted(doc) ? doc + : SortPageXML.sorted(doc); + List regions = (filter == null) + ? new ElementList(sorted.getElementsByTagName("TextRegion")) + : filter.selectElements(sorted); + + for (int r = 0; r < regions.size(); ++r) { + Element region = regions.get(r); + readPageTextRegion(region); + } + } + + /** + * Reads textual content from FR10 XML paragraph + * + * @param oar + * the paragraph (par) element + */ + private void readFR10Par(Element par) throws WarningException { + NodeList lines = par.getElementsByTagName("line"); + for (int nline = 0; nline < lines.getLength(); ++nline) { + Element line = (Element) lines.item(nline); + StringBuilder text = new StringBuilder(); + NodeList formattings = line.getElementsByTagName("formatting"); + for (int nform = 0; nform < formattings.getLength(); ++nform) { + Element formatting = (Element) formattings.item(nform); + NodeList charParams = formatting.getElementsByTagName("charParams"); + for (int nchar = 0; nchar < charParams.getLength(); ++nchar) { + Element charParam = (Element) charParams.item(nchar); + String content = charParam.getTextContent(); + if (content.length() > 0) { + text.append(content); + } else { + text.append(' '); + } + } + } + add(text.toString()); + } + } + + /** + * Reads textual content from FR10 XML file + * + * @param file + * the input XML file + */ + private void readFR10File(File file) throws WarningException { + Document doc = loadXMLFile(file); + + List pars = (filter == null) + ? new ElementList(doc.getElementsByTagName("par")) + : filter.selectElements(doc); + + for (int npar = 0; npar < pars.size(); ++npar) { + Element par = pars.get(npar); + readFR10Par(par); + } + } + + /** + * Reads textual content from HOCR HTML file + * + * @param file + * the input HTML file + */ + private void readHOCRFile(File file) throws WarningException { + try { + org.jsoup.nodes.Document doc = org.jsoup.Jsoup.parse(file, null); + Charset htmlEncoding = doc.outputSettings().charset(); + + if (htmlEncoding == null) { + encoding = Encoding.detect(file); + Messages.warning("No charset declaration in " + + file + ". Using " + encoding); + } else { + encoding = htmlEncoding; + Messages.info("HTML file " + file + + " encoding is " + encoding); + } + + for (org.jsoup.nodes.Element e : doc.body().select( + "*[class=ocr_line")) { + String text = e.text(); + add(text); + + } + } catch (IOException ex) { + Messages.info(Text.class.getName() + ": " + ex); + } + } + + /** + * Reads textual content in ALTO XML element of type TextLine + * + * @param file + * the input ALTO file + */ + private void readALTOTextLine(Element line) throws WarningException { + NodeList strings = line.getElementsByTagName("String"); + + for (int nstring = 0; nstring < strings.getLength(); ++nstring) { + Element string = (Element) strings.item(nstring); + String text = string.getAttribute("CONTENT"); + add(text); + } + + } + + /** + * Reads textual content from ALTO XML file + * + * @param file + * the input ALTO file + */ + private void readALTOFile(File file) throws WarningException { + Document doc = loadXMLFile(file); + NodeList lines = doc.getElementsByTagName("TextLine"); + + for (int nline = 0; nline < lines.getLength(); ++nline) { + Element line = (Element) lines.item(nline); + readALTOTextLine(line); + } + } + + /** + * Extract text content (under the filtered elements) + * + * @throws java.io.IOException + */ + public static void main(String[] args) throws IOException, + WarningException, XPathExpressionException, SchemaLocationException { + if (args.length < 1 | args[0].equals("-h")) { + System.err.println("usage: Text xmlfile [xpathfile] [xpathfile]"); + } else { + File xmlfile = new File(args[0]); + File inclusions = args.length > 1 ? new File(args[1]) : null; + File exclusions = args.length > 2 ? new File(args[2]) : null; + XPathFilter filter = (inclusions == null) + ? null + : new XPathFilter(inclusions, exclusions); + + Text text = new Text(xmlfile, null, filter); + System.out.println(text); + } + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/text/UnicodeReader.java b/ocrevalUAtion/src/main/java/eu/digitisation/text/UnicodeReader.java new file mode 100644 index 00000000..1ce17c81 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/text/UnicodeReader.java @@ -0,0 +1,164 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.text; + +import eu.digitisation.log.Messages; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.PrintWriter; + + +/** + * Transformations between Unicode strings and codepoints + * + * @version 2012.06.20 + */ +public class UnicodeReader { + + /** + * Transform a sequence of Unicode values (blocks of four hexadecimal + * digits) into the string they represent. For example, "00410042" (or "0041 + * 0042") represents "AB" + * + * @param codes the sequence of one or more Unicode hex values + * @return the string represented by these codes + * @throws java.io.IOException + */ + protected static String codepointsToString(String codes) throws IOException { + StringBuilder builder = new StringBuilder(); + String[] tokens = codes.trim().split("\\p{Space}+"); + + for (String token : tokens) { + if (token.length() % 4 != 0) { + throw new IOException(token + + " is not a valid Unicode hex sequence"); + } + for (int pos = 0; pos + 3 < token.length(); pos += 4) { + String sub = token.substring(pos, pos + 4); + int val = Integer.parseInt(sub, 16); + builder.append((char) val); + } + } + return builder.toString(); + } + + /** + * Build a string from the codepoints (Unicode values) defining its content + * + * @param codes + * @return the string represented by those codepoints + */ + public static String codepointsToString(int[] codes) { + StringBuilder buff = new StringBuilder(); + for (int code : codes) { + buff.append((char) code); + } + return buff.toString(); + } + + /** + * Convert a string into a sequence of Unicode values + * + * @param s a Java String + * @return The array of Unicode values of the characters in s + */ + public static int[] toCodepoints(String s) { + int[] codes = new int[s.length()]; + for (int n = 0; n < s.length(); ++n) { + codes[n] = (int) s.charAt(n); + } + return codes; + } + + /** + * Transform an array of integers into their hexadecimal representation + * + * @param values an integer array + * @return the hexadecimal strings representing their value. + */ + private static String[] toHexString(int[] values) { + String[] hex = new String[values.length]; + for (int n = 0; n < values.length; ++n) { + hex[n] = Integer.toHexString(values[n]); + } + return hex; + } + + /** + * Convert a string into a sequence of Unicode hexadecimal values + * + * @param s a Java String + * @return The array of Unicode values (hexadecimal representation) of the + * characters in s + */ + public static String[] toHexCodepoints(String s) { + return toHexString(toCodepoints(s)); + } + + /** + * Read a text file and print the content as codepoints (Unicode values) in + * it + * + * @param file the input file + * @throws Exception + */ + public static void printHexCodepoints(File file) + throws Exception { + BufferedReader reader = new BufferedReader(new FileReader(file)); + while (reader.ready()) { + String line = reader.readLine(); + String[] hexcodes = toHexCodepoints(line); + System.out.println(java.util.Arrays.toString(hexcodes)); + } + reader.close(); + } + + /** + * Search for a Unicode sequence and highlight them in browser + * + * @param files files where the sequence is searched + * @param outFile the output file + * @param codepoints the Unicode sequence + */ + public static void find(File[] files, String codepoints, File outFile) { + try { + String pattern = codepointsToString(codepoints); + for (File file : files) { + BufferedReader reader = new BufferedReader(new FileReader(file)); + PrintWriter writer = new PrintWriter(outFile); + writer.print("

" + + file + "

"); + while (reader.ready()) { + String line = reader.readLine(); + if (line.contains(pattern)) { + writer.print("

" + + StringNormalizer.encode(line) + "

"); + } else { + writer.print("

" + StringNormalizer.encode(line) + "

"); + } + } + reader.close(); + writer.close(); + } + } catch (IOException ex) { + Messages.info(CharFilter.class.getName() + ": " + ex); + } + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/text/WordScanner.java b/ocrevalUAtion/src/main/java/eu/digitisation/text/WordScanner.java new file mode 100644 index 00000000..67773865 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/text/WordScanner.java @@ -0,0 +1,209 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.text; + +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.Charset; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A simple and fast text scanner which reads words from a file and performs the + * tokenization oriented by information-retrieval requirements. + * + * @version 2012.06.20 + */ +public class WordScanner { + + static Pattern pattern; + Matcher matcher; + BufferedReader reader; + + static { + StringBuilder builder = new StringBuilder(); + builder.append("("); + builder.append("(\\p{L}+([-\\x26'+/@·.]\\p{L}+)*)"); + builder.append("|"); + builder.append("([\\p{Nd}\\p{Nl}\\p{No}]+([-',./][\\p{Nd}\\p{Nl}\\p{No}]+)*[%‰]?)"); + builder.append(")"); + pattern = Pattern.compile(builder.toString()); + } + + /** + * Open an InputStream for scanning + * + * @param is the InputStream + * @param encoding the character encoding (e.g., UTF-8). + * @param regex the regular expression for words + * @throws IOException + */ + public WordScanner(InputStream is, Charset encoding, String regex) + throws IOException { + InputStreamReader isr = new InputStreamReader(is, encoding); + + if (regex != null) { + pattern = Pattern.compile(regex); + } + + reader = new BufferedReader(isr); + if (reader.ready()) { + matcher = pattern.matcher(reader.readLine()); + } else { + matcher = pattern.matcher(""); + } + } + + /** + * Open an InputStream for scanning + * + * @param is the InputStream + * @param encoding the character encoding (e.g., UTF-8). + * @throws IOException + */ + public WordScanner(InputStream is, Charset encoding) + throws IOException { + this(is, encoding, null); + } + + /** + * Open file with specific encoding for scanning. + * + * @param file the input file. + * @param encoding the encoding (e.g., UTF-8). + * @param regex the regular expression for words + * @throws java.io.IOException + */ + public WordScanner(File file, Charset encoding, String regex) + throws IOException { + this(new FileInputStream(file), encoding, regex); + } + + /** + * Open file with specific encoding for scanning. + * + * @param file the input file. + * @param encoding the encoding (e.g., UTF-8). + * @throws java.io.IOException + */ + public WordScanner(File file, Charset encoding) + throws IOException { + this(new FileInputStream(file), encoding); + } + + /** + * Open a file for scanning. + * + * @param file the input file. + * @throws java.io.IOException + */ + public WordScanner(File file, String regex) + throws IOException { + this(file, Encoding.detect(file), regex); + } + + /** + * Open a string for scanning + * + * @param s the input string to be tokenized + * @param regex the regular expression for words + * @throws IOException + */ + public WordScanner(String s, String regex) throws IOException { + this(new ByteArrayInputStream(s.getBytes("UTF-8")), + Charset.forName("UTF-8"), regex); + } + + /** + * Open a string for scanning + * + * @param s the input string to be tokenized + * @throws IOException + */ + public WordScanner(String s) throws IOException { + this(new ByteArrayInputStream(s.getBytes("UTF-8")), + Charset.forName("UTF-8")); + } + + /** + * + * @param file the input file to be processed + * @param regex the regular expression for words + * @return a StringBuilder with the file content + * @throws java.io.IOException + */ + public static StringBuilder scanToStringBuilder(File file, String regex) + throws IOException { + StringBuilder builder = new StringBuilder(); + WordScanner scanner = new WordScanner(file, regex); + String word; + + while ((word = scanner.nextWord()) != null) { + builder.append(' ').append(word); + } + return builder; + } + + /** + * Returns the next word in file. + * + * @return the next word in the scanned file + * @throws java.io.IOException + */ + public String nextWord() + throws IOException { + String res = null; + while (res == null) { + if (matcher.find()) { + res = matcher.group(0); + } else if (reader.ready()) { + matcher = pattern.matcher(reader.readLine()); + } else { + break; + } + } + return res; + } + + /** + * Sample main. + * + * @param args + */ + public static void main(String[] args) { + WordScanner scanner; + + for (String arg : args) { + try { + String word; + File file = new File(arg); + scanner = new WordScanner(file, "[^\\p{Space}]+"); + while ((word = scanner.nextWord()) != null) { + System.out.println(word); + } + } catch (IOException ex) { + System.err.println(ex); + } + } + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/text/WordSet.java b/ocrevalUAtion/src/main/java/eu/digitisation/text/WordSet.java new file mode 100644 index 00000000..bf29fa29 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/text/WordSet.java @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.text; + +import eu.digitisation.input.WarningException; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.HashSet; + +/** + * + * @author R.C.C. + */ +public class WordSet extends HashSet { + + private static final long serialVersionUID = 1L; + + /** + * Default constructor + * + * @param file the file containing the list of stop-words (separated by + * blanks or newlines) + * @throws IOException + */ + public WordSet(File file) throws WarningException { + try { + BufferedReader reader = new BufferedReader(new FileReader(file)); + while (reader.ready()) { + String line = reader.readLine().trim(); + for (String word : line.split("\\p{Space}+")) { + if (word.length() > 0) { + add(word); + } + } + } + } catch (IOException ex) { + throw new WarningException("File " + file + + " is not a valid stop word file"); + } + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/xml/DocumentBuilder.java b/ocrevalUAtion/src/main/java/eu/digitisation/xml/DocumentBuilder.java new file mode 100644 index 00000000..d63a4909 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/xml/DocumentBuilder.java @@ -0,0 +1,203 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.xml; + +import eu.digitisation.log.Messages; +import java.util.ArrayList; +import java.util.List; +import javax.xml.parsers.ParserConfigurationException; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +/** + * Adds some useful auxiliary functions to handle XML documents + * + * @author R.C.C + */ +public class DocumentBuilder { + + Document doc; + + /** + * Create an empty document + * + * @param doctype the document type + */ + public DocumentBuilder(String doctype) { + try { + doc = javax.xml.parsers.DocumentBuilderFactory + .newInstance().newDocumentBuilder() + .newDocument(); + Element root = doc.createElement(doctype); + doc.appendChild(root); + } catch (ParserConfigurationException ex) { + Messages.info(DocumentBuilder.class.getName() + ": " + ex); + } + } + + /** + * Create a copy of another document + * + * @param other + */ + public DocumentBuilder(Document other) { + doc = clone(other); + } + + /** + * Create a copy of another document + * + * @param source a source document + * @return a deep copy of the document + */ + public static Document clone(Document source) { + try { + Document target = javax.xml.parsers.DocumentBuilderFactory + .newInstance().newDocumentBuilder() + .newDocument(); + Node node = target.importNode(source.getDocumentElement(), true); + target.appendChild(node); + return target; + } catch (ParserConfigurationException ex) { + Messages.info(DocumentBuilder.class.getName() + ": " + ex); + } + return null; + } + + /** + * + * @return the org.w3c.dom.Document + */ + public Document document() { + return doc; + } + + /** + * + * @return the root element in this document + */ + public Element root() { + return doc.getDocumentElement(); + } + + /** + * + * @param e The parent element + * @param name The child element name + * @return list of children of e with the given tag + */ + public static List getChildElementsByTagName(Element e, String name) { + ArrayList list = new ArrayList(); + NodeList children = e.getChildNodes(); + + for (int n = 0; n < children.getLength(); ++n) { + Node node = children.item(n); + if (node instanceof Element && node.getNodeName().equals(name)) { + list.add((Element) node); + } + } + return list; + } + + /** + * Create a new element under the designated element in the document. + * + * @param parent the parent element + * @param tag The tag of the new child element + * @return the added element + */ + public Element addElement(Element parent, String tag) { + Element element = doc.createElement(tag); + parent.appendChild(element); + return element; + } + + /** + * Create a new element directly under the root element. + * + * @param tag The tag of the new child element + * @return the added element + */ + public Element addElement(String tag) { + return addElement(root(), tag); + } + + /** + * Insert an element as a child of another element + * + * @param parent the parent element + * @param child the child element (even external one) + * @return the parent element + */ + public Element addElement(Element parent, Element child) { + if (parent.getOwnerDocument() == child.getOwnerDocument()) { + parent.appendChild(child); + } else { + parent.appendChild(doc.importNode(child, true)); + } + return parent; + } + + /** + * Add text content under the given element + * + * @param parent the container element + * @param content the textual content + */ + public void addText(Element parent, String content) { + parent.appendChild(doc.createTextNode(content)); + } + + /** + * Add a text element with the specified textual content under the + * designated element in the document. + * + * @param parent the parent element + * @param tag the new child element tag + * @param content the textual content + * @return the added element + */ + public Element addTextElement(Element parent, String tag, String content) { + Element element = doc.createElement(tag); + element.appendChild(doc.createTextNode(content)); + parent.appendChild(element); + return element; + } + + /** + * Dump content to string + * + * @return the content as a string + */ + @Override + public String toString() { + return new DocumentWriter(doc).toString(); + } + + /** + * Dump content to file + * + * @param file the output file + */ + public void write(java.io.File file) { + new DocumentWriter(doc).write(file); + } + +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/xml/DocumentParser.java b/ocrevalUAtion/src/main/java/eu/digitisation/xml/DocumentParser.java new file mode 100644 index 00000000..73f9dc3d --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/xml/DocumentParser.java @@ -0,0 +1,79 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.xml; + +import eu.digitisation.log.Messages; +import java.io.IOException; +import javax.xml.parsers.ParserConfigurationException; +import org.w3c.dom.Document; +import org.xml.sax.SAXException; + +/** + * A builder and parser for XML documents + * + * @author R.C.C. + * @version 2011.03.10 + */ +public class DocumentParser { + + static javax.xml.parsers.DocumentBuilder docBuilder; + + static { + try { + docBuilder = javax.xml.parsers.DocumentBuilderFactory + .newInstance().newDocumentBuilder(); + } catch (ParserConfigurationException ex) { + Messages.info(DocumentParser.class.getName() + ": " + ex); + } + } + + /** + * Create XML document from file content + * + * @param file the input file + * @return an XML document + */ + public static Document parse(java.io.File file) { + try { + return docBuilder.parse(file); + } catch (SAXException ex) { + Messages.info(DocumentParser.class.getName() + ": " + ex); + } catch (IOException ex) { + Messages.info(DocumentParser.class.getName() + ": " + ex); + } + return null; + } + + /** + * Create XML document from InputStream content + * + * @param is the InputStream with XML content + * @return an XML document + */ + public static Document parse(java.io.InputStream is) { + try { + return docBuilder.parse(is); + } catch (SAXException ex) { + Messages.info(DocumentParser.class.getName() + ": " + ex); + } catch (IOException ex) { + Messages.info(DocumentParser.class.getName() + ": " + ex); + } + + return null; + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/xml/DocumentWriter.java b/ocrevalUAtion/src/main/java/eu/digitisation/xml/DocumentWriter.java new file mode 100644 index 00000000..08faf9b8 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/xml/DocumentWriter.java @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.xml; + +import eu.digitisation.log.Messages; +import javax.xml.transform.TransformerConfigurationException; +import javax.xml.transform.TransformerException; +import org.w3c.dom.Document; + +/** + * Writes XML document to String or File + * + * @author R.C.C. + */ +public class DocumentWriter { + + static javax.xml.transform.Transformer transformer; + + javax.xml.transform.dom.DOMSource source; + javax.xml.transform.stream.StreamResult result; + + static { + try { + transformer = javax.xml.transform.TransformerFactory + .newInstance().newTransformer(); + } catch (TransformerConfigurationException ex) { + Messages.info(DocumentWriter.class.getName() + ": " + ex); + } + } + + /** + * Create a DocumentWriter for a given document + * + * @param document the XML document + */ + public DocumentWriter(Document document) { + source = new javax.xml.transform.dom.DOMSource(document); + } + + /** + * Write XML to string + * + * @return string representation + */ + @Override + public String toString() { + result = new javax.xml.transform.stream.StreamResult(new java.io.StringWriter()); + try { + transformer.transform(source, result); + } catch (TransformerException ex) { + Messages.info(DocumentParser.class.getName() + ": " + ex); + } + return result.getWriter().toString(); + } + + /** + * Dump content to file + * + * @param file the output file + */ + public void write(java.io.File file) { + result = new javax.xml.transform.stream.StreamResult(file); + try { + transformer.transform(source, result); + } catch (TransformerException ex) { + Messages.info(DocumentParser.class.getName() + ": " + ex); + } + + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/xml/ElementList.java b/ocrevalUAtion/src/main/java/eu/digitisation/xml/ElementList.java new file mode 100644 index 00000000..20c4bf89 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/xml/ElementList.java @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.xml; + +import java.util.ArrayList; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +/** + * A class transforming a org.w3c.dom.NodeList into a list of elements + * + * @author R.C.C + */ +public class ElementList extends ArrayList { + + /** + * Create a list of elements from a org.w3c.dom.NodeList + * + * @param nodes + */ + public ElementList(NodeList nodes) { + for (int n = 0; n < nodes.getLength(); ++n) { + Node node = nodes.item(n); + if (node instanceof Element) { + add((Element) node); + } else { + throw new IllegalArgumentException("ElementList: " + + "source NodeList contains non-element nodes"); + } + } + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/xml/XML2text.java b/ocrevalUAtion/src/main/java/eu/digitisation/xml/XML2text.java new file mode 100644 index 00000000..d886126a --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/xml/XML2text.java @@ -0,0 +1,185 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.xml; + +import eu.digitisation.input.Parameters; +import eu.digitisation.log.Messages; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.FilenameFilter; +import java.io.IOException; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.parsers.SAXParserFactory; +import org.xml.sax.Attributes; +import org.xml.sax.SAXException; +import org.xml.sax.XMLReader; +import org.xml.sax.helpers.DefaultHandler; + +/** + * Removes markup, declarations, PI's and comments from XML files. Implemented + * as a SAX parser. + */ +public class XML2text extends DefaultHandler { + + static final String helpMsg = "Usage:\t" + + "java -cp target/ocrevaluation.jar eu.digitisation.xml.XML2text -in file1" + + " -o output_file_or_dir"; + + private static void exit_gracefully() { + System.err.println(helpMsg); + System.exit(0); + } + + private StringBuilder buffer; + private static final FilenameFilter filter = new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.endsWith(".xml"); + } + }; + + @Override + public void characters(char[] c, int start, int length) { + if (length > 0) { + buffer.append(c, start, length); + } + } + + @Override + public void startElement(String uri, String localName, + String tag, Attributes attributes) { + buffer.append(" "); + } + + @Override + public void endElement(String uri, String localName, String tag) { + buffer.append(" "); + } + + private XMLReader getXMLReader() { + XMLReader reader = null; + try { + reader = SAXParserFactory.newInstance() + .newSAXParser().getXMLReader(); + reader.setContentHandler(this); + + } catch (SAXException ex) { + Messages.info(XML2text.class.getName() + ": " + ex); + } catch (ParserConfigurationException ex) { + Messages.info(XML2text.class.getName() + ": " + ex); + } + return reader; + } + + /** + * Read file and return text content. + * + * @param fileName the name of the file. + * @return text in file without markup. + */ + public String getText(String fileName) { + XMLReader reader = getXMLReader(); + buffer = new StringBuilder(10000); + try { + reader.parse(fileName); + } catch (IOException ex) { + Messages.info(XML2text.class.getName() + ": " + ex); + } catch (SAXException ex) { + Messages.info(XML2text.class.getName() + ": " + ex); + } + return buffer.toString(); + } + + /** + * Main function + * + * @param args + * @throws IOException + */ + public static void main(String[] args) throws IOException { + XML2text xml = new XML2text(); + String outDir = null; + Parameters pars = new Parameters(); + + for (int n = 0; n < args.length; ++n) { + String arg = args[n]; + if (arg.equals("-o")) { + pars.outfile.setValue(new File(args[++n])); + } else if (arg.equals("-in")) { + pars.ocrfile.setValue(new File(args[++n])); + }else { + System.err.println("Unrecognized option " + arg); + System.err.println("usage: XML2text [-o outfile]" + + "file1.xml file2.xml..."); + } + } + + if (pars.ocrfile.getValue() == null || pars.ocrfile.getValue() == null) { + System.err.println("Not enough arguments"); + exit_gracefully(); + } + + if (pars.outfile.getValue() == null) { + String name = pars.ocrfile.getValue().getName().replaceAll("\\.\\w+", "") + + ".txt"; + pars.outfile.setValue(new File(pars.ocrfile.getValue().getParent(), name)); + exit_gracefully(); + } + + File infile = new File(pars.ocrfile.getValue().toString()); + String outfileName = pars.outfile.getValue().toString(); + + File outfile = new File(outDir, outfileName); + if (outfile.exists()) { + System.err.println(outfileName + "already exists "); + } else { + BufferedWriter writer = + new BufferedWriter(new FileWriter(outfileName)); + writer.write(xml.getText(infile.getAbsolutePath())); + writer.close(); + } + //pars.outfile.getValue() + + + /* + for (int n = 0; n < args.length; ++n) { + String arg = args[n]; + + if (arg.equals("-d")) { + outDir = args[++n]; + if (! new File(outDir).mkdir()) { + throw new IOException("Unable to create dir " + outDir); + } + } else if (arg.endsWith(".xml")) { + File infile = new File(arg); + String outfileName = arg.replace(".xml", ".txt"); + File outfile = new File(outDir, outfileName); + if (outfile.exists()) { + System.err.println(outfileName + "already exists "); + } else { + BufferedWriter writer = + new BufferedWriter(new FileWriter(outfileName)); + writer.write(xml.getText(infile.getAbsolutePath())); + writer.close(); + } + } + } + */ + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/xml/XMLPath.java b/ocrevalUAtion/src/main/java/eu/digitisation/xml/XMLPath.java new file mode 100644 index 00000000..9752aa11 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/xml/XMLPath.java @@ -0,0 +1,131 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.xml; + +import eu.digitisation.log.Messages; +import java.io.File; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpressionException; +import javax.xml.xpath.XPathFactory; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; + +/** + * Evaluate XPath expressions. + */ +public class XMLPath { + + final static XPath xpath; + + static { + xpath = XPathFactory.newInstance().newXPath(); + } + + /** + * Evaluate XPath expression + * + * @param doc the container document + * @param expression XPath expression + * @return the list of nodes matching the query + */ + public static NodeList evaluate(Document doc, String expression) { + try { + return (NodeList) xpath.evaluate(expression, doc, + XPathConstants.NODESET); + } catch (XPathExpressionException ex) { + Messages.info(XMLPath.class.getName() + ": " + ex); + } + return null; + } + + /** + * Evaluate XPath expression + * + * @param file the file containing the XML document + * @param expression XPath expression + * @return the list of nodes matching the query + */ + public static NodeList evaluate(File file, String expression) { + Document doc = DocumentParser.parse(file); + try { + return (NodeList) xpath.evaluate(expression, doc, + XPathConstants.NODESET); + } catch (XPathExpressionException ex) { + Messages.info(XMLPath.class.getName() + ": " + ex); + } + return null; + } + + /** + * + * @param element an XML element + * @param expression an XPath expression + * @return the list of descendent nodes matching the query + */ + public static NodeList evaluate(Element element, String expression) { + try { + return (NodeList) xpath.evaluate(expression, element, + XPathConstants.NODESET); + } catch (XPathExpressionException ex) { + Messages.info(XMLPath.class.getName() + ": " + ex); + } + return null; + } + + /** + * Test if an element matches the given expression + * + * @param element an XML element + * @param expression an XPath expression with respect to the element it self + * (e.g., an element tag) + * @return true if the given element matches the query + */ + public static boolean matches(Element element, String expression) { + try { + return (Boolean) xpath.evaluate("self::" + expression, element, XPathConstants.BOOLEAN); + } catch (XPathExpressionException ex) { + Messages.info(XMLPath.class.getName() + ": " + ex); + } + return false; + } + + /** + * Sample main + * + * @param args + */ + public static void main(String[] args) { + if (args.length != 2) { + System.err.println("usage: XMLpath filename xpath"); + } else { + File file = new File(args[0]); + String expr = args[1]; + try { + NodeList nodes = XMLPath.evaluate(file, expr); + System.out.println(nodes.getLength()); + for (int n = 0; n < nodes.getLength(); ++n) { + System.out.println(nodes.item(n)); + } + } catch (Exception e) { + System.err.println(e); + } + } + } +} diff --git a/ocrevalUAtion/src/main/java/eu/digitisation/xml/XPathFilter.java b/ocrevalUAtion/src/main/java/eu/digitisation/xml/XPathFilter.java new file mode 100644 index 00000000..f7162703 --- /dev/null +++ b/ocrevalUAtion/src/main/java/eu/digitisation/xml/XPathFilter.java @@ -0,0 +1,265 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.xml; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpression; +import javax.xml.xpath.XPathExpressionException; +import javax.xml.xpath.XPathFactory; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +/** + * Test elements against XPath expressions and include or exclude the elements + * + * @author R.C.C. + */ +public class XPathFilter { + + static XPath xpath = XPathFactory.newInstance().newXPath(); + List inexpr; // the inclussion expressions + List exexpr; // the exclussion expressions + List inclusions; // XPath expressions of included elements + List exclusions; // XPath expressions of excluded elements + + private void include(String expression) throws XPathExpressionException { + inexpr.add(expression); + inclusions.add(xpath.compile("self::" + expression)); + } + + private void includeAll(String[] array) throws XPathExpressionException { + if (array != null) { + for (String s : array) { + include(s); + } + } + } + + private void exclude(String expression) throws XPathExpressionException { + exexpr.add(expression); + exclusions.add(xpath.compile("self::" + expression)); + } + + private void excludeAll(String[] array) throws XPathExpressionException { + if (array != null) { + for (String s : array) { + exclude(s); + } + } + } + + /** + * Default constructor + */ + public XPathFilter() { + inexpr = new ArrayList(); + exexpr = new ArrayList(); + inclusions = new ArrayList(); + exclusions = new ArrayList(); + } + + /** + * Create an XPathFilter from two arrays of XPath expressions + * + * @param inclusions an array of XPath inclusion expressions (possibly null) + * @param exclusions array of XPath exclusion expressions (possibly null) + * @throws javax.xml.xpath.XPathExpressionException + */ + public XPathFilter(String[] inclusions, String[] exclusions) + throws XPathExpressionException { + this(); + includeAll(inclusions); + excludeAll(exclusions); + } + + /** + * Read file into lines + * + * @param file the input file + * @return the content as a list of strings, each one with the content in + * one file line, excluding those starting with the character # + * @throws IOException + */ + private String[] lines(File file) throws IOException { + List list = new ArrayList(); + BufferedReader reader = new BufferedReader(new FileReader(file)); + while (reader.ready()) { + String line = reader.readLine().trim(); + if (line.length() > 0 && !line.startsWith("#")) { + list.add(line); + } + } + return list.toArray(new String[list.size()]); + } + + /** + * Create an ElementFilter from the XPath expressions in a file (one per + * line) + * + * @param infile a file containing XPath inclusion expressions (one per + * line) + * @param exfile a file containing XPath exclusion expressions (one per + * line) + * @throws java.io.IOException + * @throws javax.xml.xpath.XPathExpressionException + */ + public XPathFilter(File infile, File exfile) + throws IOException, XPathExpressionException { + this(); + if (infile != null) { + includeAll(lines(infile)); + } + if (exfile != null) { + excludeAll(lines(exfile)); + } + } + + /** + * Check if the element matches any valid inclusion expression + * + * @param element an XML element + * @return true if the element matches any of the XPath inclusion + * expressions + */ + public boolean included(Element element) { + + for (int n = 0; n < inclusions.size(); ++n) { + XPathExpression expression = inclusions.get(n); + try { + Boolean match = (Boolean) expression.evaluate(element, + XPathConstants.BOOLEAN); + if (match) { + return true; + } + } catch (XPathExpressionException ex) { + // not a valid match + } + } + return false; + } + + /** + * Check if the element matches any valid inclusion expression + * + * @param element an XML element + * @return true if the element matches any of the XPAth exclusion + * expressions + */ + public boolean excluded(Element element) { + for (XPathExpression expression : exclusions) { + try { + Boolean match = (Boolean) expression.evaluate(element, + XPathConstants.BOOLEAN); + if (match) { + return true; + } + } catch (XPathExpressionException ex) { + // not a valid match + } + } + return false; + } + + /** + * + * @param element an XML element + * @return true if the element matches any of the XPAth inclusion + * expressions and none of the exclusion expressions + */ + private boolean accepted(Element element) { + return included(element) && !excluded(element); + } + + /** + * Select elements matching the XPath valid expression + * + * @param element a parent element + * @return all descendent elements matching at least one of the XPath + * expressions in this filter + */ + public List selectElements(Element element) { + NodeList nodeList = element.getElementsByTagName("*"); + List selection = new ArrayList(); + + for (int n = 0; n < nodeList.getLength(); n++) { + Node node = nodeList.item(n); + if (node.getNodeType() == Node.ELEMENT_NODE) { + Element e = (Element) node; + if (accepted(e)) { + selection.add(e); + } + } + } + return selection; + } + + /** + * Select elements matching the XPath valid expression + * + * @param doc a container XML document + * @return all elements in the document matching at least one of the XPath + * expressions in this filter + */ + public List selectElements(Document doc) { + NodeList nodeList = doc.getElementsByTagName("*"); + List selection = new ArrayList(); + + for (int n = 0; n < nodeList.getLength(); n++) { + Node node = nodeList.item(n); + if (node.getNodeType() == Node.ELEMENT_NODE) { + Element e = (Element) node; + if (accepted(e)) { + selection.add(e); + } + } + } + return selection; +// return selectElements(doc.getDocumentElement()); + } + + /** + * Select content under the filtered elements + * + * @param args + * @throws java.io.IOException + * @throws javax.xml.xpath.XPathExpressionException + */ + public static void main(String[] args) throws IOException, XPathExpressionException { + if (args.length < 2) { + System.err.println("usage: XPathFilter xmlfile xpathfile xpathfile"); + } else { + File xmlfile = new File(args[0]); + File xpathinfile = new File(args[1]); + File xpathexfile = new File(args[2]); + XPathFilter filter = new XPathFilter(xpathinfile, xpathexfile); + List selected = filter.selectElements(DocumentParser.parse(xmlfile)); + for (Element e : selected) { + System.out.println(e.getNodeName()); + } + } + } +} diff --git a/ocrevalUAtion/src/main/resources/Image.properties b/ocrevalUAtion/src/main/resources/Image.properties new file mode 100644 index 00000000..62a68bd6 --- /dev/null +++ b/ocrevalUAtion/src/main/resources/Image.properties @@ -0,0 +1,2 @@ +# minimal value of y-projection +line.thrshold=-0.5 \ No newline at end of file diff --git a/ocrevalUAtion/src/main/resources/UnicodeCharEquivalences.csv b/ocrevalUAtion/src/main/resources/UnicodeCharEquivalences.csv new file mode 100644 index 00000000..2ff57167 --- /dev/null +++ b/ocrevalUAtion/src/main/resources/UnicodeCharEquivalences.csv @@ -0,0 +1,54 @@ +00C6 0041 0045 +0132 0049 004A +0133 0069 006A +0153 006F 0065 +A76D 0069 0073 +E1DC 00D1 +E42C 0061 0364 +E5B8 006D 0304 +E5DC 00F1 +E665 0070 0304 +E681 0071 0304 +E682 0071 0307 +E6E2 0074 0301 +EADA 017F 0074 +EBA2 017F 0069 +EBA3 017F 006C +EBA6 017F 017F +EBA7 017F 017F 0069 +EBE3 006A 0308 +EEC4 0063 006B +EEC5 0063 0074 +EEDC 0074 007A +EFA1 0061 0065 +F161 003B +F1AC 003B +F4F9 006C 006C +F4FF 017F 0074 +F501 0063 0304 +F502 0063 0068 +F503 1EBD +F504 0067 030A +F505 0067 0304 +F506 0068 030A +F507 0070 0304 +F50A 0064 0027 +F50B 006C 0027 +F50D 0071 0301 A76B +F50E 0071 0301 +F50F 0071 0303 +F510 0072 0304 +F511 0073 0303 +F512 0074 0303 +F515 0065 0074 +F517 0063 0303 +F518 0072 0303 +F519 006D 0303 +F51E 017F 0142 +FB00 0066 0066 +FB01 0066 0069 +FB02 0066 006C +FB03 0066 0066 0069 +FB04 0066 0066 006C +FB06 0073 0074 +FEFF 0020 # Byte order mark diff --git a/ocrevalUAtion/src/main/resources/UnicodeCharEquivalences.txt b/ocrevalUAtion/src/main/resources/UnicodeCharEquivalences.txt new file mode 100644 index 00000000..862b62df --- /dev/null +++ b/ocrevalUAtion/src/main/resources/UnicodeCharEquivalences.txt @@ -0,0 +1,54 @@ +FEFF, 0020, # Byte order mark +F1AC, 003B +EFA1, 00E6 +EEC4, 0063 006B +EEC5, 0063 0074 +F502, 0063 0068 +F517, 0063 0303 +F50A, 0064 0027 +F515, 0065 0074 +FB00, 0066 0066 +FB01, 0066 0069 +FB02, 0066 006C +F504, 0067 030A +F505, 0067 0304 +F506, 0068 030A +0133, 0069 006A +A76D, 0069 0073 +EBE3, 006A 0308 +F4F9, 006C 006C +F50B, 006C 0027 +E5B8, 006D 0304 +F519, 006D 0303 +E1DC, 00D1 +E5DC, 00F1 +F50F, 0071 0303 +F510, 0072 0304 +F518, 0072 0303 +EADA, 017F 0074 +EBA2, 017F 0069 +EBA6, 017F 017F +EBA7, 017F 017F 0069 +F4FF, 017F 0074 +F511, 0073 0303 +F51E, 017F 0142 +FB06, 0073 0074 +E6E2, 0074 0301 +EEDC, 0074 007A +F512, 0074 0303 +F161, 003B +F503, 1EBD +F507, 0070 0304 +00C6, 0041 0045 +0132, 0049 004A +0153, 006F 0065 +E42C, 0061 0364 +E665, 0070 0304 +E681, 0071 0304 +E682, 0071 0307 +EBA3, 017F 006C +F501, 0063 0304 +F50D, 0071 0301 A76B +F50E, 0071 0301 +FB03, 0066 0066 0069 +FB04, 0066 0066 006C \ No newline at end of file diff --git a/ocrevalUAtion/src/main/resources/defaultProperties.xml b/ocrevalUAtion/src/main/resources/defaultProperties.xml new file mode 100644 index 00000000..cc985325 --- /dev/null +++ b/ocrevalUAtion/src/main/resources/defaultProperties.xml @@ -0,0 +1,23 @@ + + + + + + 0 + + + + http://schema.primaresearch.org/PAGE/gts/pagecontent/2010-03-19 + http://schema.primaresearch.org/PAGE/gts/pagecontent/2010-03-19/pagecontent.xsd + http://schema.primaresearch.org/PAGE/gts/pagecontent/2013-07-15 + http://schema.primaresearch.org/PAGE/gts/pagecontent/2013-07-15/pagecontent.xsd + + + http://www.abbyy.com/FineReader_xml/FineReader10-schema-v1.xml + + + http://www.loc.gov/standards/alto/alto-v2.0.xsd + http://schema.ccs-gmbh.com/ALTO + + \ No newline at end of file diff --git a/ocrevalUAtion/src/test/java/eu/digitisation/distance/ArrayEditDistanceTest.java b/ocrevalUAtion/src/test/java/eu/digitisation/distance/ArrayEditDistanceTest.java new file mode 100644 index 00000000..c643e233 --- /dev/null +++ b/ocrevalUAtion/src/test/java/eu/digitisation/distance/ArrayEditDistanceTest.java @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +package eu.digitisation.distance; + +import static org.junit.Assert.assertEquals; +import org.junit.Test; + +/** + * + * @author R.C.C + */ +public class ArrayEditDistanceTest { + + /** + * Test of indel method, of class ArrayEditDistance. + */ + @Test + public void testIndelDistance() { + System.out.println("indelDistance"); + Object[] first = {'p','a','t','a','t','a'}; + Object[] second = {'a','p','t','a'}; + int expResult = 4; + int result = ArrayEditDistance.indel(first, second); + assertEquals(expResult, result); + } + + /** + * Test of levenshtein method, of class ArrayEditDistance. + */ + @Test + public void testLevenshteinDistance() { + System.out.println("levenshteinDistance"); + Object[] first = {'p','a','t','a','t','a'}; + Object[] second = {'a','p','t','a'}; + int expResult = 3; + int result = ArrayEditDistance.levenshtein(first, second); + assertEquals(expResult, result); + } + +} diff --git a/ocrevalUAtion/src/test/java/eu/digitisation/distance/EditDistanceTest.java b/ocrevalUAtion/src/test/java/eu/digitisation/distance/EditDistanceTest.java new file mode 100644 index 00000000..2a41fe6c --- /dev/null +++ b/ocrevalUAtion/src/test/java/eu/digitisation/distance/EditDistanceTest.java @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2014 Uni. de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.distance; + +import eu.digitisation.text.StringNormalizer; +import eu.digitisation.text.Text; +import java.io.File; +import java.net.URL; +import static org.junit.Assert.assertEquals; +import org.junit.Test; + +/** + * + * @author rafa + */ +public class EditDistanceTest { + + @Test + public void testWeights() { + System.out.println("Weighted distance"); + EdOpWeight w = new OcrOpWeight(); + String s1 = "a b"; + String s2 = "acb"; + int expResult = 2; + int result = EditDistance.charDistance(s1, s2, w, 50); + assertEquals(expResult, result); + } + + /** + * Test of wordDistance method, of class EditDistance. + */ + @Test + public void testWordDistance() { + System.out.println("wordDistance"); + String s1 = "p a t a t a"; + String s2 = "a p t a"; + int expResult = 3; + int[] result = EditDistance.wordDistance(s1, s2, 10); + assertEquals(expResult, result[2]); + } + + @Test + public void testWeightedDistance() { + String s1 = "ÁÁÁÁ"; + String s2 = "ÁAáa"; + + OcrOpWeight W = new OcrOpWeight(); // fully-sensitive + String r1 = StringNormalizer.canonical(s1, false, false, false); + String r2 = StringNormalizer.canonical(s2, false, false, false); + assertEquals(3, EditDistance.charDistance(r1, r2, W, 1000)); + + W = new OcrOpWeight(true); //ignore everything + r1 = StringNormalizer.canonical(s1, true, true, true); + r2 = StringNormalizer.canonical(s2, true, true, true); + assertEquals(0, EditDistance.charDistance(r1, r2, W, 1000)); + + W = new OcrOpWeight(true); //ignore diacritics + r1 = StringNormalizer.canonical(s1, false, true, true); + r2 = StringNormalizer.canonical(s2, false, true, true); + assertEquals(2, EditDistance.charDistance(r1, r2, W, 1000)); + + W = new OcrOpWeight(true); //ignore case + r1 = StringNormalizer.canonical(s1, true, false, true); + r2 = StringNormalizer.canonical(s2, true, false, true); + assertEquals(2, EditDistance.charDistance(r1, r2, W, 1000)); + } +} diff --git a/ocrevalUAtion/src/test/java/eu/digitisation/distance/EditSequenceTest.java b/ocrevalUAtion/src/test/java/eu/digitisation/distance/EditSequenceTest.java new file mode 100644 index 00000000..33d5948a --- /dev/null +++ b/ocrevalUAtion/src/test/java/eu/digitisation/distance/EditSequenceTest.java @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.distance; + +import eu.digitisation.math.BiCounter; +import eu.digitisation.output.CharStatTable; +import static org.junit.Assert.assertEquals; +import org.junit.Test; + +/** + * + * @author rafa + */ +public class EditSequenceTest { + + /** + * Test of cost method, of class EditSequence. + */ + @Test + public void testCost() { + System.out.println("cost"); + EditSequence instance = new EditSequence("acb", "a b", new OcrOpWeight()); + int expResult = 2; + int result = instance.length(); + assertEquals(expResult, result); + } + + /** + * Test of shift1 method, of class EditSequence. + */ + @Test + public void testShift1() { + System.out.println("shift1"); + EditSequence instance = new EditSequence("acb", "a b", new OcrOpWeight()); + int expResult = 3; + int result = instance.shift1(); + assertEquals(expResult, result); + } + + @Test + public void testStats() { + System.out.println("stats"); + String s1 = "acb"; + String s2 = "abs"; + EditSequence instance = new EditSequence(s1, s2, new OcrOpWeight()); + BiCounter result = instance.stats(s1, s2); + BiCounter expResult = new BiCounter(); + expResult.add('a', EdOp.KEEP, 1); + expResult.add('b', EdOp.KEEP, 1); + expResult.add('c', EdOp.DELETE, 1); + expResult.add('s', EdOp.INSERT, 1); + assertEquals(expResult, result); + + EdOpWeight w = new OcrOpWeight(); + result = instance.stats(s1, s2, w); + assertEquals(expResult, result); + } + + @Test + public void testPunct() { + OcrOpWeight w = new OcrOpWeight(true); /// ignore punctuation + String s1 = "yes ! , he said"; + String s2 = "yes he said"; + EditSequence edit = new EditSequence(s1, s2, new OcrOpWeight()); + CharStatTable stats = new CharStatTable(); + stats.add(edit.stats(s1, s2)); + System.out.println(stats); + double cer = stats.cer(); + + assertEquals(0, 0, 0.00001); + } +} diff --git a/ocrevalUAtion/src/test/java/eu/digitisation/distance/EditTableTest.java b/ocrevalUAtion/src/test/java/eu/digitisation/distance/EditTableTest.java new file mode 100644 index 00000000..8246152b --- /dev/null +++ b/ocrevalUAtion/src/test/java/eu/digitisation/distance/EditTableTest.java @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.distance; + +import static org.junit.Assert.assertEquals; +import org.junit.Test; + +/** + * + * @author R.C.C + */ +public class EditTableTest { + + public EditTableTest() { + } +/* + @Test + public void testSet() { + System.out.println("set"); + byte b = 0; + byte result = EditTable.setBit(b, 0, true); + System.out.println(result); + assertEquals(1, result); + assertEquals(true, EditTable.getBit(result,0)); + } +*/ + /** + * Test of get method, of class EditTable. + */ + @Test + public void testGet() { + System.out.println("get"); + EditTable instance = new EditTable(2, 2); + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < 2; ++j) { + if (i == j) { + instance.set(i, j, EdOp.KEEP); + } else { + instance.set(i, j, EdOp.SUBSTITUTE); + } + } + } + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < 2; ++j) { + if (i == j) { + assertEquals(EdOp.KEEP, instance.get(i, j)); + } else { + assertEquals(EdOp.SUBSTITUTE, instance.get(i, j)); + } + } + } + } +} diff --git a/ocrevalUAtion/src/test/java/eu/digitisation/distance/OcrOpWeightTest.java b/ocrevalUAtion/src/test/java/eu/digitisation/distance/OcrOpWeightTest.java new file mode 100644 index 00000000..3542f635 --- /dev/null +++ b/ocrevalUAtion/src/test/java/eu/digitisation/distance/OcrOpWeightTest.java @@ -0,0 +1,89 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.distance; + +import eu.digitisation.text.StringNormalizer; +import static org.junit.Assert.assertEquals; +import org.junit.Test; + +/** + * + * @author rafa + */ +public class OcrOpWeightTest { + + /** + * Test of sub method, of class OcrWeights. + */ + @Test + public void testSub() { + System.out.println("sub"); + char[] c1 = {'Á', 'Á', 'Á', 'Á', 'Á'}; + char[] c2 = {'Á', 'A', 'á', 'a', ' '}; + int[] w1 = {0, 1, 1, 1, 2}; + OcrOpWeight W1 = new OcrOpWeight(); // fully-sensitive + for (int n = 0; n < w1.length; ++n) { + String s1 = StringNormalizer.canonical(String.valueOf(c1[n]), false, false, false); + String s2 = StringNormalizer.canonical(String.valueOf(c2[n]), false, false, false); + int d = EditDistance.charDistance(s1, s2, W1, 10); + assertEquals(w1[n], d); + } + OcrOpWeight W2 = new OcrOpWeight(true); //ignore everything + int[] w2 = {0, 0, 0, 0, 2}; + for (int n = 0; n < w2.length; ++n) { + String s1 = StringNormalizer.canonical(String.valueOf(c1[n]), true, true, true); + String s2 = StringNormalizer.canonical(String.valueOf(c2[n]), true, true, true); + int d = EditDistance.charDistance(s1, s2, W2, 10); + assertEquals(w2[n], d); + } + OcrOpWeight W3 = new OcrOpWeight(false); //ignore diacritics + int[] w3 = {0, 0, 1, 1, 2}; + for (int n = 0; n < w3.length; ++n) { + String s1 = StringNormalizer.canonical(String.valueOf(c1[n]), false, true, true); + String s2 = StringNormalizer.canonical(String.valueOf(c2[n]), false, true, true); + int d = EditDistance.charDistance(s1, s2, W3, 10); + assertEquals(w3[n], d); + } + + OcrOpWeight W4 = new OcrOpWeight(true); //ignore case + int[] w4 = {0, 1, 0, 1, 2}; + for (int n = 0; n < w4.length; ++n) { + String s1 = StringNormalizer.canonical(String.valueOf(c1[n]), true, false, true); + String s2 = StringNormalizer.canonical(String.valueOf(c2[n]), true, false, true); + int d = EditDistance.charDistance(s1, s2, W4, 10); + assertEquals(w4[n], d); + } + } + + /** + * Test of ins method, of class OcrWeights. + */ + @Test + public void testIns() { + System.out.println("ins"); + OcrOpWeight W = new OcrOpWeight(true); //ignore punct + assertEquals(1, W.ins('a')); + assertEquals(0, W.ins('@')); + assertEquals(0, W.ins('+')); + W = new OcrOpWeight(); + assertEquals(1, W.ins('a')); + assertEquals(1, W.ins('@')); + assertEquals(1, W.ins('+')); + + } +} diff --git a/ocrevalUAtion/src/test/java/eu/digitisation/distance/StringEditDistanceTest.java b/ocrevalUAtion/src/test/java/eu/digitisation/distance/StringEditDistanceTest.java new file mode 100644 index 00000000..63f9fa69 --- /dev/null +++ b/ocrevalUAtion/src/test/java/eu/digitisation/distance/StringEditDistanceTest.java @@ -0,0 +1,103 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.distance; + +import eu.digitisation.text.StringNormalizer; +import eu.digitisation.math.BiCounter; +import static junit.framework.TestCase.assertEquals; +import org.junit.Test; + +/** + * + * @author R.C.C + */ +public class StringEditDistanceTest { + + public StringEditDistanceTest() { + } + + /** + * Test of indel method, of class StringEditDistance. + */ + @Test + public void testIndelDistance() { + System.out.println("indelDistance"); + String first = "patata"; + String second = "apta"; + int expResult = 4; + int result = StringEditDistance.indel(first, second); + assertEquals(expResult, result); + + } + + /** + * Test of levenshtein method, of class StringEditDistance. + */ + @Test + public void testLevenshteinDistance() { + System.out.println("levenshteinDistance"); + String first = "patata"; + String second = "apta"; + int expResult = 3; + int result = StringEditDistance.levenshtein(first, second); + assertEquals(expResult, result); + // A second test + first = "holanda"; + second = "wordland"; + result = StringEditDistance.levenshtein(first, second); + assertEquals(4, result); + // Test with normalization + first = StringNormalizer.reduceWS("Mi enhorabuena"); + second = StringNormalizer.reduceWS("mi en hora buena"); + result = StringEditDistance.levenshtein(first, second); + assertEquals(3, result); + } + + + /** + * Test of DL method, of class StringEditDistance. + */ + @Test + public void testDLDistance() { + System.out.println("Damerau-Levenshtein Distance"); + String first = "abracadabra"; + String second = "arbadacarba"; + int expResult = 4; + int result = StringEditDistance.DL(first, second); + assertEquals(expResult, result); + + } + + @Test + public void testOperations() { + System.out.println("operations"); + String first = "patata"; + String second = "apta"; + BiCounter expResult = new BiCounter(); + expResult.add('a', EdOp.KEEP, 2); // sure + expResult.inc('t', EdOp.KEEP); // sure + expResult.inc('p', EdOp.DELETE); // sure + expResult.inc('t', EdOp.SUBSTITUTE); // not the ony pssibility + expResult.inc('a', EdOp.DELETE); // could exchange with 'a' above + + BiCounter result + = StringEditDistance.operations(first, second); + System.out.println(result); + assertEquals(expResult, result); + } +} diff --git a/ocrevalUAtion/src/test/java/eu/digitisation/distance/WordCompareTest.java b/ocrevalUAtion/src/test/java/eu/digitisation/distance/WordCompareTest.java new file mode 100644 index 00000000..a2ac7b3b --- /dev/null +++ b/ocrevalUAtion/src/test/java/eu/digitisation/distance/WordCompareTest.java @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.distance; + +import static org.junit.Assert.assertEquals; +import org.junit.Test; + +/** + * + * @author rafa + */ +public class WordCompareTest { + + /** + * Test of wdiff method, of class WordCompare. + */ + @Test + public void testWdiff() { + System.out.println("wdiff"); + String first = "one to just one"; + String second = "to see one"; + String expResult = "one # []\nto = to\njust # see\none = one\n"; + String result = WordCompare.wdiff(first, second); + assertEquals(expResult, result); + } + +} diff --git a/ocrevalUAtion/src/test/java/eu/digitisation/document/TermFrequencyVectorTest.java b/ocrevalUAtion/src/test/java/eu/digitisation/document/TermFrequencyVectorTest.java new file mode 100644 index 00000000..aa9138c3 --- /dev/null +++ b/ocrevalUAtion/src/test/java/eu/digitisation/document/TermFrequencyVectorTest.java @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.document; + +import eu.digitisation.distance.EditDistance; +import static org.junit.Assert.assertEquals; +import org.junit.Test; + +/** + * + * @author rafa + */ +public class TermFrequencyVectorTest { + + String s1 = //"UN AN : PARIS, 8 Francs. — PROVINCE, 10 Francs. — ETRANGER, suivant le Tarif postal. " + "A LA LIBRAIRIE, 10, RUE DE LA BOURSE. CHRONIQUE GOURMANDE UNE des gracieuses"; + String s2 = //"V AN : PA's»s*c8fFrancs. — Pr«vjnv-e, 11 > Fr.it:-*.— K: kvnobi. ', Tarif ;\".s:a!. 1- ni 7 + "A LA LIBRAIRIE, 10. RUE DE LA BOURSE. TOUS PREMIER. I I VRAIS'< , CHRONIQUE GOURMANDE * ,' -~J.,' 1 Ii nk .!•'« gracieuses"; + + public TermFrequencyVectorTest() { + + } + + /** + * Test of distance method, of class TermFrequencyVector. + */ + @Test + public void testDistance() { + System.out.println("distance"); + TermFrequencyVector tf1 = new TermFrequencyVector(s1); + TermFrequencyVector tf2 = new TermFrequencyVector(s2); + int expResult = EditDistance.wordDistance(s1, s2, 1000)[2]; + int result = tf1.distance(tf2); + assertEquals(expResult, result); + } + + /** + * Test of total method, of class TermFrequencyVector. + */ + @Test + public void testTotal() { + System.out.println("total"); + TermFrequencyVector tf1 = new TermFrequencyVector(s1); + TermFrequencyVector tf2 = new TermFrequencyVector(s2); + System.out.println("tf1=" + tf1.toString()); + System.out.println("f2=" + tf2.toString()); + assertEquals(13, tf1.total()); + assertEquals(20, tf2.total()); + } + +} diff --git a/ocrevalUAtion/src/test/java/eu/digitisation/document/TokenArrayTest.java b/ocrevalUAtion/src/test/java/eu/digitisation/document/TokenArrayTest.java new file mode 100644 index 00000000..8b9b2b91 --- /dev/null +++ b/ocrevalUAtion/src/test/java/eu/digitisation/document/TokenArrayTest.java @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.document; + +import eu.digitisation.math.MinimalPerfectHash; +import static junit.framework.TestCase.assertEquals; +import org.junit.Test; + +/** + * + * @author R.C.C + */ +public class TokenArrayTest { + + public TokenArrayTest() { + } + + + + /** + * Test of encode method, of class TextFileEncoder. + */ + @Test + public void testEncode_String() { + System.out.println("encode"); + String input = "hola&amigo2\n3.14 mi casa, todos los días\n" + + "mesa-camilla java4you i.b.m. i+d Dª María 3+100%"; + String expOutput = "hola&amigo 2 3.14 mi casa todos los días" + + " mesa-camilla java 4 you i.b.m i+d Dª María 3 100%"; + MinimalPerfectHash f = new MinimalPerfectHash(true); + TokenArray array = new TokenArray(f, input); + String output = array.toString(); + assertEquals(expOutput, output); + + int size = array.length(); + assertEquals(18, size); + } +} diff --git a/ocrevalUAtion/src/test/java/eu/digitisation/input/BatchTest.java b/ocrevalUAtion/src/test/java/eu/digitisation/input/BatchTest.java new file mode 100644 index 00000000..a53602a7 --- /dev/null +++ b/ocrevalUAtion/src/test/java/eu/digitisation/input/BatchTest.java @@ -0,0 +1,83 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.input; + +import eu.digitisation.input.Batch; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * + * @author R.C.C + */ +public class BatchTest { + + public BatchTest() { + } + + @BeforeClass + public static void setUpClass() { + } + + @AfterClass + public static void tearDownClass() { + } + + @Before + public void setUp() { + } + + @After + public void tearDown() { + } + + /** + * Test of prefix method, of class Batch. + */ + @Test + public void testLcp() { + System.out.println("lcp"); + String s1 = "compare"; + String s2 = "competence"; + String expResult = "comp"; + String result = Batch.prefix(s1, s2); + assertEquals(expResult, result); + } + + /** + * Test of suffix method, of class Batch. + */ + @Test + public void testLcs() { + System.out.println("lcs"); + String s1 = "switzerland"; + String s2 = "disneyland"; + String expResult = "land"; + String result = Batch.suffix(s1, s2); + assertEquals(expResult, result); + s2 = "sweden"; + expResult = ""; + result = Batch.suffix(s1, s2); + assertEquals(expResult, result); + } + +} diff --git a/ocrevalUAtion/src/test/java/eu/digitisation/layout/BoundingBoxTest.java b/ocrevalUAtion/src/test/java/eu/digitisation/layout/BoundingBoxTest.java new file mode 100644 index 00000000..3375a094 --- /dev/null +++ b/ocrevalUAtion/src/test/java/eu/digitisation/layout/BoundingBoxTest.java @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.layout; + +import java.awt.Polygon; +import static org.junit.Assert.assertEquals; +import org.junit.Test; + +/** + * + * @author R.C.C + */ +public class BoundingBoxTest { + + /** + * Test of asPolygon method, of class BoundingBox. + */ + @Test + public void testToPolygon() { + System.out.println("toPolygon"); + Polygon expResult = new BoundingBox(0, 0, 20, 20).asPolygon(); + BoundingBox instance = new BoundingBox(0, 0, 10, 20); + instance.add(new BoundingBox(10, 10, 20, 20)); + + assertEquals(expResult.getBounds(), instance); + } + +} diff --git a/ocrevalUAtion/src/test/java/eu/digitisation/layout/ComponentTagTest.java b/ocrevalUAtion/src/test/java/eu/digitisation/layout/ComponentTagTest.java new file mode 100644 index 00000000..f05e330a --- /dev/null +++ b/ocrevalUAtion/src/test/java/eu/digitisation/layout/ComponentTagTest.java @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +package eu.digitisation.layout; + +import eu.digitisation.input.FileType; +import static org.junit.Assert.assertEquals; +import org.junit.Test; + +/** + * + * @author R.C.C + */ +public class ComponentTagTest { + + /** + * Test of shortTag method, of class ComponentTag. + */ + @Test + public void testShortTag() { + System.out.println("shortTag"); + ComponentTag tag = ComponentTag.valueOf(FileType.PAGE, "TextRegion"); + String expResult = "TextRegion"; + String result = ComponentTag.shortTag(tag); + assertEquals(expResult, result); + } + +} diff --git a/ocrevalUAtion/src/test/java/eu/digitisation/math/ArraysTest.java b/ocrevalUAtion/src/test/java/eu/digitisation/math/ArraysTest.java new file mode 100644 index 00000000..efdc2b40 --- /dev/null +++ b/ocrevalUAtion/src/test/java/eu/digitisation/math/ArraysTest.java @@ -0,0 +1,212 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.math; + +import static org.junit.Assert.assertEquals; +import org.junit.Test; + +/** + * + * @author rafa + */ +public class ArraysTest { + + /** + * Test of sum method, of class ArrayMath. + */ + @Test + public void testSum_intArr() { + System.out.println("sum"); + int[] array = {1, 2, 3, 0, -1}; + int expResult = 5; + int result = Arrays.sum(array); + assertEquals(expResult, result); + } + + /** + * Test of sum method, of class ArrayMath. + */ + @Test + public void testSum_doubleArr() { + System.out.println("sum"); + double[] array = {1, 2, 3, 0, -1}; + double expResult = 5; + double result = Arrays.sum(array); + assertEquals(expResult, result, 0.01); + } + + /** + * Test of average method, of class ArrayMath. + */ + @Test + public void testAverage_intArr() { + System.out.println("average"); + int[] array = {1, 2, 3, -2}; + double expResult = 1.0; + double result = Arrays.average(array); + assertEquals(expResult, result, 0.0001); + } + + /** + * Test of average method, of class ArrayMath. + */ + @Test + public void testAverage_doubleArr() { + System.out.println("average"); + double[] array = {1, 2, 3, -2}; + double expResult = 1.0; + double result = Arrays.average(array); + assertEquals(expResult, result, 0.0001); + } + + /** + * Test of logaverage method, of class ArrayMath. + */ + @Test + public void testLogaverage_intArr() { + System.out.println("logaverage"); + int[] array = {10, 100, 1000}; + double expResult = 100.0; + double result = Arrays.logaverage(array); + assertEquals(expResult, result, 0.001); + } + + /** + * Test of logaverage method, of class ArrayMath. + */ + @Test + public void testLogaverage_doubleArr() { + System.out.println("logaverage"); + double[] array = {10, 100, 1000}; + double expResult = 100.0; + double result = Arrays.logaverage(array); + assertEquals(expResult, result, 0.001); + } + + /** + * Test of scalar method, of class ArrayMath. + */ + @Test + public void testScalar() { + System.out.println("scalar"); + double[] x = {1, 2, 3}; + double[] y = {1, 2, 3}; + double expResult = 14.0; + double result = Arrays.scalar(x, y); + assertEquals(expResult, result, 0.0001); + } + + /** + * Test of max method, of class ArrayMath. + */ + @Test + public void testMax_intArr() { + System.out.println("max"); + int[] array = {-5, 2, 3}; + int expResult = 3; + int result = Arrays.max(array); + assertEquals(expResult, result); + } + + /** + * Test of max method, of class ArrayMath. + */ + @Test + public void testMax_doubleArr() { + System.out.println("max"); + double[] array = {-5, 2, 3}; + double expResult = 3; + double result = Arrays.max(array); + assertEquals(expResult, result, 0.0); + } + + /** + * Test of min method, of class ArrayMath. + */ + @Test + public void testMin_intArr() { + System.out.println("min"); + int[] array = {2, -1}; + int expResult = -1; + int result = Arrays.min(array); + assertEquals(expResult, result); + } + + /** + * Test of min method, of class ArrayMath. + */ + @Test + public void testMin_doubleArr() { + System.out.println("min"); + double[] array = {2, 0, -1}; + double expResult = -1.0; + double result = Arrays.min(array); + assertEquals(expResult, result, 0.0001); + } + + /** + * Test of cov method, of class ArrayMath. + */ + @Test + public void testCov_intArr_intArr() { + System.out.println("cov"); + int[] X = {1, 2, 3}; + int[] Y = {1, 2, 3}; + double expResult = 2.0/3; + double result = Arrays.cov(X, Y); + assertEquals(expResult, result, 0.001); + } + + /** + * Test of cov method, of class ArrayMath. + */ + @Test + public void testCov_doubleArr_doubleArr() { + System.out.println("cov"); + double[] X = {1, 2, 3}; + double[] Y = {1, 2, 3}; + double expResult = 2.0/3; + double result = Arrays.cov(X, Y); + assertEquals(expResult, result, 0.001); + } + + /** + * Test of std method, of class ArrayMath. + */ + @Test + public void testStd_intArr() { + System.out.println("std"); + int[] X = {1, 2, 2, 3}; + double expResult = Math.sqrt(0.5); + double result = Arrays.std(X); + assertEquals(expResult, result, 0.0001); + } + + /** + * Test of std method, of class ArrayMath. + */ + @Test + public void testStd_doubleArr() { + System.out.println("std"); + double[] X = {1, 2, 2, 3}; + double expResult = Math.sqrt(0.5); + double result = Arrays.std(X); + assertEquals(expResult, result, 0.0001); + } + +} diff --git a/ocrevalUAtion/src/test/java/eu/digitisation/math/BiCounterTest.java b/ocrevalUAtion/src/test/java/eu/digitisation/math/BiCounterTest.java new file mode 100644 index 00000000..2cbd358c --- /dev/null +++ b/ocrevalUAtion/src/test/java/eu/digitisation/math/BiCounterTest.java @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.math; + +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * + * @author R.C.C + */ +public class BiCounterTest { + + public BiCounterTest() { + } + + @BeforeClass + public static void setUpClass() { + } + + @AfterClass + public static void tearDownClass() { + } + + @Before + public void setUp() { + } + + @After + public void tearDown() { + } + + /** + * Test of value method, of class BiCounter. + */ + @Test + public void testValue() { + System.out.println("value"); + Object o1 = null; + Object o2 = null; + BiCounter bc = new BiCounter(); + bc.inc(1, 2); + bc.inc(1, 3); + bc.add(1, 3, 4); + assertEquals(1, bc.value(1, 2)); + assertEquals(5, bc.value(1, 3)); + assertEquals(6, bc.value(1, null)); + assertEquals(6, bc.total()); + } + +} diff --git a/ocrevalUAtion/src/test/java/eu/digitisation/math/CounterTest.java b/ocrevalUAtion/src/test/java/eu/digitisation/math/CounterTest.java new file mode 100644 index 00000000..6c0e9232 --- /dev/null +++ b/ocrevalUAtion/src/test/java/eu/digitisation/math/CounterTest.java @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.math; + +import java.util.List; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * + * @author R.C.C + */ +public class CounterTest { + + public CounterTest() { + } + + @BeforeClass + public static void setUpClass() { + } + + @AfterClass + public static void tearDownClass() { + } + + @Before + public void setUp() { + } + + @After + public void tearDown() { + } + + /** + * Test of add method, of class Counter. + */ + @Test + public void test() { + System.out.println("add"); + Object object = null; + int value = 0; + Counter instance = new Counter(); + instance.add(1, 3); + instance.inc(1); + instance.add(1, -1); + assertEquals(instance.get(1).intValue(), 3); + } + + @Test + public void testKeyList() { + System.out.println("keyList"); + Counter instance = new Counter(); + instance.add(1, 6); + instance.add(2, 3); + instance.add(3, 1); + instance.add(4, 5); + Integer[] expResult = {3, 2, 4, 1}; + Integer[] result = new Integer[4]; + List list = instance.keyList(Counter.Order.ASCENDING_VALUE); + System.out.println(list); + list.toArray(result); + assertArrayEquals(expResult, result); + } + +} diff --git a/ocrevalUAtion/src/test/java/eu/digitisation/math/PairTest.java b/ocrevalUAtion/src/test/java/eu/digitisation/math/PairTest.java new file mode 100644 index 00000000..6b0eefef --- /dev/null +++ b/ocrevalUAtion/src/test/java/eu/digitisation/math/PairTest.java @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.math; + +import org.junit.Test; + +/** + * + * @author R.C.C + */ +public class PairTest { + /** + * Test of equals method, of class Pair. + */ + @Test + public void testEquals() { + System.out.println("equals"); + Object o = null; + Pair p1 = new Pair("a", "b"); + Pair p2 = new Pair("a", "b"); + Pair p3 = new Pair("a", "c"); + assert (p1.equals(p2)); + assert (!p1.equals(p3)); + } +} diff --git a/ocrevalUAtion/src/test/java/eu/digitisation/ngram/NgramModelTest.java b/ocrevalUAtion/src/test/java/eu/digitisation/ngram/NgramModelTest.java new file mode 100644 index 00000000..988f91d9 --- /dev/null +++ b/ocrevalUAtion/src/test/java/eu/digitisation/ngram/NgramModelTest.java @@ -0,0 +1,200 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.ngram; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import static org.junit.Assert.assertEquals; +import org.junit.Test; + +/** + * + * @author carrasco@ua.es + */ +public class NgramModelTest { + + @Test + public void testSize() { + System.out.println("size"); + NgramModel ngrams = new NgramModel(2); + ngrams.addWord("0000"); + ngrams.addWord("0100"); + + int expResult = 9; // 5 bi-grams plus 3 uni-grams plus 1 0-gram + int result = ngrams.size(); + + assertEquals(expResult, result); + + ngrams = new NgramModel(3); + ngrams.addWord("0000"); + ngrams.addWord("0100"); + + expResult = 15; // 6 tri-grams + 5 bi-grams +3 uni-grams + 1 0-gram + result = ngrams.size(); + + assertEquals(expResult, result); + + } + + @Test + public void testGetGoodTuringPars() { + System.out.println("size"); + NgramModel ngrams = new NgramModel(3); + ngrams.addWord("0000"); + ngrams.addWord("0100"); + double[] expResult = {0.1, 0.2, 0.5}; + double[] result = ngrams.getGoodTuringPars(); + assertEquals(expResult.length, result.length); + for (int n = 0; n < result.length; ++n) { + assertEquals(expResult[n], result[n], 0.001); + } + + } + + @Test + public void testProb() { + System.out.println("prob"); + NgramModel ngrams = new NgramModel(3); + ngrams.addWord("0000"); + ngrams.addWord("0100"); + + assertEquals(4 / (double) 7, ngrams.prob("00"), 0.001); + assertEquals(0.7, ngrams.prob("0"), 0.001); + } + + @Test + public void testSmoothProb() { + System.out.println("prob"); + NgramModel ngrams = new NgramModel(3); + ngrams.addWord("0000"); + ngrams.addWord("0100"); + + double expResult = 0.8 * (4 / (double) 7) + 0.2 * 0.7; + double result = ngrams.smoothProb("00"); + assertEquals(expResult, result, 0.001); + + expResult = 0.8 * (2 / (double) 7) + 0.2 * 0.2; + result = ngrams.smoothProb("0" + ngrams.EOS); + assertEquals(expResult, result, 0.001); + } + + @Test + public void testWordLogProb() { + System.out.println("wordLogProb"); + NgramModel instance = new NgramModel(1); + instance.addWord("lava"); + double expResult = (3 * Math.log(0.2) + 2 * Math.log(0.4)); + double result = instance.logWordProb("lava"); + assertEquals(expResult, result, 0.01); + } + + @Test + public void testLogProb() { + System.out.println("logProb"); + NgramModel instance = new NgramModel(1); + instance.addWord("lava"); + double expResult = -Math.log(5); + double result = instance.logProb("baba", 'v'); + assertEquals(expResult, result, 0.01); + + instance = new NgramModel(2); + instance.addWord("lava"); + expResult = Math.log(0.2); + result = instance.logProb("ca", 'v'); + assertEquals(expResult, result, 0.01); + } + + @Test + public void testAddSubstrings() { + System.out.println("addSubstrings"); + String EOS = String.valueOf(NgramModel.EOS); + NgramModel ngrams = new NgramModel(3); + NgramModel ref = new NgramModel(3); + + ngrams.addSubstrings("b", "cde"); + + // 3-grams +// ref.addEntry("abc"); + ref.addEntry("bcd"); + ref.addEntry("cde"); + + // 2-grams + ref.addEntry("bc"); + ref.addEntry("cd"); + ref.addEntry("de"); + + // 1-grams + ref.addEntry("c"); + ref.addEntry("d"); + ref.addEntry("e"); + + // 0-grams + ref.addEntries("", 3); + ref.showDiff(ngrams); + + assertEquals(ref, ngrams); + + } + + @Test + public void testAddText() { + System.out.println("addText"); + String BOS = String.valueOf(NgramModel.BOS); + String EOS = String.valueOf(NgramModel.EOS); + NgramModel ngrams = new NgramModel(3); + NgramModel ref = new NgramModel(3); + String input = "ab\nc"; + InputStream is = new ByteArrayInputStream(input.getBytes()); + + // result + ngrams.addText(is); + + // expected result + // 3-grams + ref.addEntry(BOS + "ab"); + ref.addEntry("ab "); + ref.addEntry("b c"); + ref.addEntry(" c" + EOS); + + //2-grams + ref.addEntry(BOS + 'a'); + ref.addEntry("ab"); + ref.addEntry("b "); + ref.addEntry(" c"); + ref.addEntry("c" + EOS); + // 1-grams + ref.addEntry("a"); + ref.addEntry("b"); + ref.addEntry(" "); + ref.addEntry("c"); + ref.addEntry(EOS); + // 0-grams + ref.addEntries("", 5); + + ref.showDiff(ngrams); + + assertEquals(ref, ngrams); + + String text = "ab"; + + is = new ByteArrayInputStream(text.getBytes()); + double expectedResult = Math.log(0.2); + double result = ngrams.logLikelihood(is, 0); + assertEquals(expectedResult, result, 0.0001); + } +} diff --git a/ocrevalUAtion/src/test/java/eu/digitisation/text/CharFilterTest.java b/ocrevalUAtion/src/test/java/eu/digitisation/text/CharFilterTest.java new file mode 100644 index 00000000..abe94bd0 --- /dev/null +++ b/ocrevalUAtion/src/test/java/eu/digitisation/text/CharFilterTest.java @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.text; + +import eu.digitisation.text.CharFilter; +import java.io.File; +import java.net.URISyntaxException; +import java.net.URL; +import static org.junit.Assert.assertEquals; +import org.junit.Test; + +/** + * + * @author R.C.C + */ +public class CharFilterTest { + + public CharFilterTest() { + } + + + /** + * Test of translate method, of class CharFilter. + * + * @throws java.net.URISyntaxException + */ + @Test + public void testTranslate_String() throws URISyntaxException { + System.out.println("translate"); + URL resourceUrl = getClass().getResource("/UnicodeCharEquivalences.txt"); + File file = new File(resourceUrl.toURI()); + CharFilter filter = new CharFilter(file); + String s = "a\u0133"; // ij + String expResult = "aij"; + String result = filter.translate(s); + assertEquals(expResult.length(), result.length()); + assertEquals(expResult, result); + } + + @Test + public void testCompatibilityMode() { + System.out.println("compatibility"); + CharFilter filter = new CharFilter(); + String s = "\u0133"; + String r = "ij"; + assert (!r.equals(filter.translate(s))); + filter.setCompatibility(true); + assertEquals(r, filter.translate(s)); + + } +} diff --git a/ocrevalUAtion/src/test/java/eu/digitisation/text/CharMapTest.java b/ocrevalUAtion/src/test/java/eu/digitisation/text/CharMapTest.java new file mode 100644 index 00000000..d9dac59c --- /dev/null +++ b/ocrevalUAtion/src/test/java/eu/digitisation/text/CharMapTest.java @@ -0,0 +1,96 @@ +/* + * Copyright (C) 2014 U. de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.text; + +import eu.digitisation.text.CharMap; +import eu.digitisation.text.CharMap.Option; +import java.io.File; +import java.net.URISyntaxException; +import java.net.URL; +import static org.junit.Assert.assertEquals; +import org.junit.Test; + +/** + * + * @author rafa + */ +public class CharMapTest { + + public CharMapTest() { + } + + /** + * Test of normalForm method, of class CharMap. + * @throws java.net.URISyntaxException + */ + @Test + public void testNormalForm_String() throws URISyntaxException { + System.out.println("normalForm"); + String s = "Mañana! Antígona2"; + Option[] ops = {}; + CharMap map = new CharMap(ops); + String expResult = "manana antigona2"; + String result = map.normalForm(s); + assertEquals(expResult, result); + // Test comaptiblity file + URL resourceUrl = getClass().getResource("/UnicodeCharEquivalences.txt"); + File file = new File(resourceUrl.toURI()); + CharMap filter = new CharMap(); + filter.addFilter(file); + s = "a\uf50d"; // triple ligature not in Unicode compatibilty + expResult = "aq\u0301\uA76B"; // q + acute + et + result = filter.normalForm(s); + assertEquals(expResult, result); + } + + /** + * Test of normalForm method, of class CharMap. + */ + @Test + public void testNormalForm_char() { + System.out.println("normalForm"); + char longs = '\u017F'; // a long s + char ff = '\ufb00'; + Option[] ops = {}; + CharMap map = new CharMap(ops); + String expResult; + String result; + result = map.normalForm(longs); + expResult = "s"; + assertEquals(expResult, result); + result = map.normalForm(ff); + expResult = "ff"; + assertEquals(expResult, result); + } + + /** + * Test of equiv method, of class CharMap. + */ + @Test + public void testEquiv() { + System.out.println("equiv"); + char c1 = '?'; + char c2 = ' '; + Option[] ops = {}; + CharMap instance = new CharMap(ops); + boolean expResult = true; + boolean result = instance.equiv(c1, c2); + assertEquals(expResult, result); + } + +} diff --git a/ocrevalUAtion/src/test/java/eu/digitisation/text/StringNormalizerTest.java b/ocrevalUAtion/src/test/java/eu/digitisation/text/StringNormalizerTest.java new file mode 100644 index 00000000..96f58927 --- /dev/null +++ b/ocrevalUAtion/src/test/java/eu/digitisation/text/StringNormalizerTest.java @@ -0,0 +1,122 @@ +/* + * Copyright (C) 2014 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.text; + +import static org.junit.Assert.assertEquals; +import org.junit.Test; + +/** + * + * @author rafa + */ +public class StringNormalizerTest { + + public StringNormalizerTest() { + } + + /** + * Test of reduceWS method, of class StringNormalizer. + */ + @Test + public void testReduceWS() { + System.out.println("reduceWS"); + String s = "one \rtwo\nthree"; + String expResult = "one two three"; + String result = StringNormalizer.reduceWS(s); + assertEquals(expResult, result); + } + + /** + * Test of composed method, of class StringNormalizer. + */ + @Test + public void testComposed() { + System.out.println("composed"); + String s = "n\u0303"; + String expResult = "ñ"; + String result = StringNormalizer.composed(s); + assertEquals(expResult, result); + } + + /** + * Test of compatible method, of class StringNormalizer. + */ + @Test + public void testCompatible() { + System.out.println("compatible"); + String s = "\ufb00"; // ff ligature + String expResult = "ff"; + String result = StringNormalizer.compatible(s); + assertEquals(expResult, result); + } + + /** + * Test of removeDiacritics method, of class StringNormalizer. + */ + @Test + public void testRemoveDiacritics() { + System.out.println("removeDiacritics"); + String s = "cañón"; + String expResult = "canon"; + String result = StringNormalizer.removeDiacritics(s); + assertEquals(expResult, result); + } + + /** + * Test of removePunctuation method, of class StringNormalizer. + */ + @Test + public void testRemovePunctuation() { + System.out.println("removePunctuation"); + String s = "!\"#}-"; // + is not in punctuation block + String expResult = ""; + String result = StringNormalizer.removePunctuation(s); + assertEquals(expResult, result); + } + + @Test + public void testTrim() { + System.out.println("trim"); + String s = "! \"#lin?ks+!\"#}-"; // + is not in punctuation block + String expResult = "lin?ks+"; + String result = StringNormalizer.trim(s); + assertEquals(expResult, result); + } + + @Test + public void testStrip() { + System.out.println("strip"); + String s = "Stra\u00dfe+ links+!\"#}-"; // ª is a letter! + String expResult = "Stra\u00dfe links"; + String result = StringNormalizer.strip(s); + assertEquals(expResult, result); + } + + /** + * Test of encode method, of class StringNormalizer. + */ + @Test + public void testEncode() { + System.out.println("encode"); + String s = "<\">"; + String expResult = "<">"; + String result = StringNormalizer.encode(s); + assertEquals(expResult, result); + + } +} diff --git a/ocrevalUAtion/src/test/java/eu/digitisation/text/TestUnicodeReader.java b/ocrevalUAtion/src/test/java/eu/digitisation/text/TestUnicodeReader.java new file mode 100644 index 00000000..f45ef4f7 --- /dev/null +++ b/ocrevalUAtion/src/test/java/eu/digitisation/text/TestUnicodeReader.java @@ -0,0 +1,20 @@ +package eu.digitisation.text; + +import eu.digitisation.text.UnicodeReader; +import junit.framework.TestCase; +/** + * + * @author R.C.C + */ +public class TestUnicodeReader extends TestCase { + + public void testUnicodeReader() { + String input = "día, mes y año"; + String ref = "[100, 237, 97, 44, 32, 109, 101, 115, 32, 121, 32, 97, 241, 111]"; + + String output = + java.util.Arrays.toString(UnicodeReader.toCodepoints(input)); + //System.out.println(output); + assertEquals(ref, output); + } +} diff --git a/ocrevalUAtion/src/test/java/eu/digitisation/text/WordScannerTest.java b/ocrevalUAtion/src/test/java/eu/digitisation/text/WordScannerTest.java new file mode 100644 index 00000000..c3aa589a --- /dev/null +++ b/ocrevalUAtion/src/test/java/eu/digitisation/text/WordScannerTest.java @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2013 Universidad de Alicante + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package eu.digitisation.text; + +import java.io.IOException; +import static junit.framework.TestCase.assertEquals; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * + * @author R.C.C + */ +public class WordScannerTest { + + public WordScannerTest() { + } + + @BeforeClass + public static void setUpClass() { + } + + @AfterClass + public static void tearDownClass() { + } + + @Before + public void setUp() { + } + + @After + public void tearDown() { + } + + /** + * Test of main method, of class WordScanner. + * + * @throws java.io.IOException + */ + @Test + public void testnextWord() throws IOException { + System.out.println("main"); + String input = "hola&amigo2\n3.14 mi casa, todos los días\n" + + "mesa-camilla java4you i.b.m. i+d Dª María 3+100%"; + WordScanner scanner = new WordScanner(input, null); + String word; + int num = 0; + while ((word = scanner.nextWord()) != null) { + ++num; + //System.out.println(word); + } + assertEquals(18, num); + + } +} diff --git a/ocrevalUAtion/userProperties.xml b/ocrevalUAtion/userProperties.xml new file mode 100644 index 00000000..0cb7bd58 --- /dev/null +++ b/ocrevalUAtion/userProperties.xml @@ -0,0 +1,9 @@ + + + + + + 100000 + + \ No newline at end of file diff --git a/ocrevalUAtion/userProperties_test.xml b/ocrevalUAtion/userProperties_test.xml new file mode 100644 index 00000000..8e486fd0 --- /dev/null +++ b/ocrevalUAtion/userProperties_test.xml @@ -0,0 +1,13 @@ + + + + + http://bibnum.bnf.fr/ns/alto_prod + http://bibnum.bnf.fr/ns/alto_prod.xsd + + + + 0 + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index 7e55dd78..3045b965 100644 --- a/pom.xml +++ b/pom.xml @@ -4,14 +4,15 @@ 4.0.0 - de.vorb + de.vorb.tesseract tesseract4java - 0.2.0-SNAPSHOT + 0.3.0-SNAPSHOT pom gui tools + ocrevalUAtion @@ -20,4 +21,36 @@ 1.8 + + + + de.vorb.tesseract + tools + ${project.version} + + + de.vorb.tesseract + ocrevalUAtion + ${project.version} + + + + + + + + com.amashchenko.maven.plugin + gitflow-maven-plugin + 1.12.0 + + 1 + true + true + false + false + + + + + diff --git a/tools/pom.xml b/tools/pom.xml index 83c2896c..7f6c3259 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -5,42 +5,41 @@ 4.0.0 - de.vorb + de.vorb.tesseract tesseract4java - 0.2.0-SNAPSHOT + 0.3.0-SNAPSHOT tools - 3.04.01-1.2 + 4.0.0-1.4.4 org.bytedeco.javacpp-presets - tesseract + tesseract-platform ${tesseract.version} net.imagej ij - 1.48v + 1.52h de.biomedical-imaging.ij ij_blob - 1.3.3 + 1.4.9 - eu.digitisation + de.vorb.tesseract ocrevalUAtion - 1.3.4-SNAPSHOT junit junit - 4.12 + 4.13.1 test @@ -48,11 +47,11 @@ central - http://repo1.maven.org/maven2 + https://repo1.maven.org/maven2 imagej - http://maven.imagej.net/content/groups/public + https://maven.imagej.net/content/groups/public diff --git a/tools/src/main/java/de/vorb/tesseract/tools/preprocessing/binarization/BinarizationUtilities.java b/tools/src/main/java/de/vorb/tesseract/tools/preprocessing/binarization/BinarizationUtilities.java index ade61c56..4882d8c7 100644 --- a/tools/src/main/java/de/vorb/tesseract/tools/preprocessing/binarization/BinarizationUtilities.java +++ b/tools/src/main/java/de/vorb/tesseract/tools/preprocessing/binarization/BinarizationUtilities.java @@ -6,31 +6,32 @@ public final class BinarizationUtilities { - private BinarizationUtilities() {} + private BinarizationUtilities() { + } private static final ColorConvertOp RGB_TO_GRAYSCALE = new ColorConvertOp( ColorSpace.getInstance(ColorSpace.CS_sRGB), ColorSpace.getInstance(ColorSpace.CS_GRAY), null); - public static BufferedImage imageToGrayscale(BufferedImage image) { + static BufferedImage imageToGrayscale(BufferedImage image) { final BufferedImage grayscale; - // return the buffered image as-is, if it is binary already - if (image.getType() == BufferedImage.TYPE_BYTE_BINARY) { - return image; - } else if (image.getType() == BufferedImage.TYPE_BYTE_GRAY) { - grayscale = image; - } else if (image.getType() == BufferedImage.TYPE_INT_RGB - || image.getType() == BufferedImage.TYPE_BYTE_INDEXED) { - grayscale = new BufferedImage(image.getWidth(), image.getHeight(), - BufferedImage.TYPE_BYTE_GRAY); - - // convert rgb image to grayscale - RGB_TO_GRAYSCALE.filter(image, grayscale); - } else { - throw new IllegalArgumentException(String.format( - "illegal color space: %s", - image.getColorModel().getColorSpace().getType())); + switch (image.getType()) { + case BufferedImage.TYPE_BYTE_BINARY: + return image; + case BufferedImage.TYPE_BYTE_GRAY: + grayscale = image; + break; + case BufferedImage.TYPE_INT_RGB: + case BufferedImage.TYPE_BYTE_INDEXED: + case BufferedImage.TYPE_3BYTE_BGR: + case BufferedImage.TYPE_4BYTE_ABGR: + grayscale = new BufferedImage(image.getWidth(), image.getHeight(), BufferedImage.TYPE_BYTE_GRAY); + RGB_TO_GRAYSCALE.filter(image, grayscale); + break; + default: + throw new IllegalArgumentException( + "illegal color space: " + image.getColorModel().getColorSpace().getType()); } return grayscale;