diff --git a/CHANGELOG.md b/CHANGELOG.md index 47ae621112..c615c97558 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,12 @@ BioJava Changelog ----------------- +BioJava 7.1.0 - future release +============================== +### Added +* Class `FastaStreamer` to read FASTA-formatted files using Java streams + + BioJava 7.0.2 ============================== ### Added diff --git a/biojava-aa-prop/pom.xml b/biojava-aa-prop/pom.xml index 908f2d28c2..84fdbf3a7e 100644 --- a/biojava-aa-prop/pom.xml +++ b/biojava-aa-prop/pom.xml @@ -2,7 +2,7 @@ biojava org.biojava - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT 4.0.0 biojava-aa-prop @@ -70,12 +70,12 @@ org.biojava biojava-core - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT org.biojava biojava-structure - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT diff --git a/biojava-alignment/pom.xml b/biojava-alignment/pom.xml index 002fbb6b0b..6ce294e7e3 100644 --- a/biojava-alignment/pom.xml +++ b/biojava-alignment/pom.xml @@ -4,7 +4,7 @@ biojava org.biojava - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT biojava-alignment biojava-alignment @@ -47,7 +47,7 @@ org.biojava biojava-core - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT compile diff --git a/biojava-core/pom.xml b/biojava-core/pom.xml index fb44bbadd3..b0e4fa55c0 100644 --- a/biojava-core/pom.xml +++ b/biojava-core/pom.xml @@ -3,7 +3,7 @@ biojava org.biojava - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT 4.0.0 biojava-core diff --git a/biojava-core/src/main/java/org/biojava/nbio/core/sequence/io/FastaStreamer.java b/biojava-core/src/main/java/org/biojava/nbio/core/sequence/io/FastaStreamer.java new file mode 100644 index 0000000000..c9184dfd13 --- /dev/null +++ b/biojava-core/src/main/java/org/biojava/nbio/core/sequence/io/FastaStreamer.java @@ -0,0 +1,179 @@ +package org.biojava.nbio.core.sequence.io; + +import org.biojava.nbio.core.sequence.ProteinSequence; +import org.biojava.nbio.core.sequence.compound.AminoAcidCompound; +import org.biojava.nbio.core.sequence.compound.AminoAcidCompoundSet; +import org.biojava.nbio.core.sequence.io.template.SequenceCreatorInterface; +import org.biojava.nbio.core.sequence.io.template.SequenceHeaderParserInterface; +import org.biojava.nbio.core.util.InputStreamProvider; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.function.Consumer; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +/** + * Read from a FASTA file (or gzipped FASTA file) and create a Java stream of {@link ProteinSequence} objects + * for use in a functional programming paradigm. + * + * @author Gary Murphy + * @since 7.1.0 + */ +public class FastaStreamer { + + private final Path path; + private int batchSize = 1_000; + private SequenceHeaderParserInterface headerParser; + private SequenceCreatorInterface sequenceCreator; + private LinkedHashMap chunk = new LinkedHashMap<>(); + private Iterator> iterator = Collections.emptyIterator(); + private boolean closed = false; + + /** + * The constructor is private. Created via the from(...) static factory method + * + * @param path the path to the file containing the FASTA content (possibly GZipped) + */ + private FastaStreamer(final Path path) { + this.path = path; + } + + public static FastaStreamer from(final Path path) { + return new FastaStreamer(path); + } + + public static FastaStreamer from(File file) { + return from(file.toPath()); + } + + public FastaStreamer withHeaderParser(SequenceHeaderParserInterface headerParser) { + this.headerParser = headerParser; + return this; + } + + public FastaStreamer withSequenceCreator(SequenceCreatorInterface sequenceCreator) { + this.sequenceCreator = sequenceCreator; + return this; + } + + public FastaStreamer batchSize(int size) { + this.batchSize = size; + return this; + } + + /** + * Enable iteration through the proteins in the file using syntax such as: + * 
+	 *     for(ProteinSequence sequence : FastaStreamer.from(path).each()) {
+	 *         .
+	 *         .
+	 *         .
+	 *     }
+	 *
+ * + * @return an iterable suitable for an iteration loop + */ + public Iterable each() { + return () -> stream().iterator(); + } + + /** + * Create a stream of protein sequences from the contents of the path + * @return the stream + */ + public Stream stream() { + InputStreamProvider provider = new InputStreamProvider(); + InputStream input; + try { + input = provider.getInputStream(getPath().toFile()); + } catch (IOException exception) { + throw new UncheckedIOException(exception); + } + FastaReader reader = new FastaReader<>(input, getHeaderParser(), getSequenceCreator()); + Spliterator source = new Spliterators.AbstractSpliterator<>(Integer.MAX_VALUE, Spliterator.IMMUTABLE | Spliterator.NONNULL) { + @Override + public boolean tryAdvance(Consumer action) { + if (closed) { + return false; + } + ProteinSequence protein = next(reader); + if (null == protein) { + return false; + } + action.accept(protein); + return true; + } + + /** + * Fetch the next header/protein tuple from the cache. If the cache is empty, fetch another + * batch from the source file + * + * @param reader + * the input stream from which the FASTA content is read + * @return the protein sequence + */ + private ProteinSequence next(FastaReader reader) { + try { + if (!iterator.hasNext()) { + chunk = reader.process(getBatchSize()); + if (null == chunk) { + closed = true; + reader.close(); + return null; + } + iterator = chunk.entrySet().iterator(); + } + if (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + return createSequence(entry.getValue()); + } + closed = true; + reader.close(); + } catch (IOException exception) { + throw new UncheckedIOException(String.format("I/O error reading the FASTA file from '%s'", getPath()), exception); + } + return null; + } + }; // Spliterator + return StreamSupport.stream(source, false); + } + + /** + * Create the sequence with the information from the header. This implementation return the sequence as-is, but + * this is an opportunity for the implementer to build specific information into the user collection space + * of the sequence + * + * @param sequence the protein sequence + * @return the sequence + */ + protected ProteinSequence createSequence(ProteinSequence sequence) { + return sequence; + } + + protected Path getPath() { + return path; + } + + protected int getBatchSize() { + return batchSize; + } + + protected SequenceHeaderParserInterface getHeaderParser() { + return Optional.ofNullable(headerParser).orElse(new GenericFastaHeaderParser<>()); + } + + public SequenceCreatorInterface getSequenceCreator() { + return Optional.ofNullable(sequenceCreator).orElse(new ProteinSequenceCreator(AminoAcidCompoundSet.getAminoAcidCompoundSet())); + } +} diff --git a/biojava-core/src/test/java/org/biojava/nbio/core/sequence/io/FastaStreamerTest.java b/biojava-core/src/test/java/org/biojava/nbio/core/sequence/io/FastaStreamerTest.java new file mode 100644 index 0000000000..614c6f06c8 --- /dev/null +++ b/biojava-core/src/test/java/org/biojava/nbio/core/sequence/io/FastaStreamerTest.java @@ -0,0 +1,50 @@ +package org.biojava.nbio.core.sequence.io; + +import org.biojava.nbio.core.sequence.ProteinSequence; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Test the functionality of the {@link FastaStreamer} code + */ +public class FastaStreamerTest { + + @Test + public void stream() throws IOException { + String file = this.getClass().getResource("PF00104_small.fasta.gz").getFile(); + Path path = Paths.get(file); + List sequences; + + sequences = FastaStreamer.from(path).stream().collect(Collectors.toList()); + Assert.assertEquals("Count", 283, sequences.size()); + + ProteinSequence sequence; + sequence = sequences.get(0); + Assert.assertEquals("A2D504_ATEGE/1-46", sequence.getOriginalHeader()); + sequence = sequences.get(sequences.size()-1); + Assert.assertEquals("Q98SJ1_CHICK/15-61", sequence.getOriginalHeader()); + + sequences = FastaStreamer.from(path) + .batchSize(2) // Ensure there isn't an edge condition loading the next buffer + .stream() + .collect(Collectors.toList()); + Assert.assertEquals("Count", 283, sequences.size()); + } + + @Test + public void iterate() { + String file = this.getClass().getResource("PF00104_small.fasta.gz").getFile(); + Path path = Paths.get(file); + int count = 0; + for (ProteinSequence sequence : FastaStreamer.from(path).each()) { + count++; + } + Assert.assertEquals("Count", 283, count); + } +} diff --git a/biojava-core/src/test/resources/org/biojava/nbio/core/sequence/io/PF00104_small.fasta.gz b/biojava-core/src/test/resources/org/biojava/nbio/core/sequence/io/PF00104_small.fasta.gz new file mode 100644 index 0000000000..d4a340c73d Binary files /dev/null and b/biojava-core/src/test/resources/org/biojava/nbio/core/sequence/io/PF00104_small.fasta.gz differ diff --git a/biojava-core/src/test/resources/org/biojava/nbio/core/util/example.gz b/biojava-core/src/test/resources/org/biojava/nbio/core/util/example.gz new file mode 100644 index 0000000000..0864558ee2 Binary files /dev/null and b/biojava-core/src/test/resources/org/biojava/nbio/core/util/example.gz differ diff --git a/biojava-genome/pom.xml b/biojava-genome/pom.xml index 2b213d7de5..f0d8e13c2f 100644 --- a/biojava-genome/pom.xml +++ b/biojava-genome/pom.xml @@ -3,7 +3,7 @@ biojava org.biojava - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT 4.0.0 biojava-genome @@ -71,13 +71,13 @@ org.biojava biojava-core - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT compile org.biojava biojava-alignment - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT compile diff --git a/biojava-integrationtest/pom.xml b/biojava-integrationtest/pom.xml index d02011571c..cee137b4c1 100644 --- a/biojava-integrationtest/pom.xml +++ b/biojava-integrationtest/pom.xml @@ -4,7 +4,7 @@ biojava org.biojava - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT biojava-integrationtest jar @@ -40,7 +40,7 @@ org.biojava biojava-structure - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT diff --git a/biojava-modfinder/pom.xml b/biojava-modfinder/pom.xml index ff8771babc..a7e07100c4 100644 --- a/biojava-modfinder/pom.xml +++ b/biojava-modfinder/pom.xml @@ -4,7 +4,7 @@ biojava org.biojava - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT biojava-modfinder biojava-modfinder @@ -31,7 +31,7 @@ org.biojava biojava-structure - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT jar compile diff --git a/biojava-ontology/pom.xml b/biojava-ontology/pom.xml index 1fb8967489..e4b3c3cdf2 100644 --- a/biojava-ontology/pom.xml +++ b/biojava-ontology/pom.xml @@ -4,7 +4,7 @@ org.biojava biojava - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT biojava-ontology diff --git a/biojava-protein-comparison-tool/pom.xml b/biojava-protein-comparison-tool/pom.xml index 36d3ec3b2b..dd4b460858 100644 --- a/biojava-protein-comparison-tool/pom.xml +++ b/biojava-protein-comparison-tool/pom.xml @@ -4,7 +4,7 @@ biojava org.biojava - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT biojava-protein-comparison-tool @@ -36,23 +36,23 @@ org.biojava biojava-alignment - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT org.biojava biojava-core - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT org.biojava biojava-structure - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT org.biojava biojava-structure-gui - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT net.sourceforge.jmol diff --git a/biojava-protein-disorder/pom.xml b/biojava-protein-disorder/pom.xml index c86173990f..67548aca4a 100644 --- a/biojava-protein-disorder/pom.xml +++ b/biojava-protein-disorder/pom.xml @@ -3,7 +3,7 @@ biojava org.biojava - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT biojava-protein-disorder jar @@ -63,7 +63,7 @@ org.biojava biojava-core - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT diff --git a/biojava-structure-gui/pom.xml b/biojava-structure-gui/pom.xml index 8b3522352b..3cc7cbe005 100644 --- a/biojava-structure-gui/pom.xml +++ b/biojava-structure-gui/pom.xml @@ -3,7 +3,7 @@ biojava org.biojava - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT 4.0.0 biojava-structure-gui @@ -27,13 +27,13 @@ org.biojava biojava-structure - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT compile org.biojava biojava-core - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT compile diff --git a/biojava-structure/pom.xml b/biojava-structure/pom.xml index bd779d4ba0..8afe66b114 100644 --- a/biojava-structure/pom.xml +++ b/biojava-structure/pom.xml @@ -4,7 +4,7 @@ biojava org.biojava - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT biojava-structure biojava-structure @@ -44,13 +44,13 @@ org.biojava biojava-alignment - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT compile org.biojava biojava-core - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT compile diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/scop/ScopInstallation.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/scop/ScopInstallation.java index b7ee585279..019e084a5d 100644 --- a/biojava-structure/src/main/java/org/biojava/nbio/structure/scop/ScopInstallation.java +++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/scop/ScopInstallation.java @@ -85,8 +85,8 @@ public class ScopInstallation implements LocalScopDatabase { public static final String comFileName = "dir.com.scop.txt_"; // Download locations - public static final String SCOP_DOWNLOAD = "http://scop.berkeley.edu/downloads/parse/"; - public static final String SCOP_DOWNLOAD_ALTERNATE = "http://scop.berkeley.edu/downloads/parse/"; + public static final String SCOP_DOWNLOAD = "https://scop.berkeley.edu/downloads/parse/"; + public static final String SCOP_DOWNLOAD_ALTERNATE = "https://scop.berkeley.edu/downloads/parse/"; //public static final String NEWLINE = System.getProperty("line.separator"); public static final String FILESPLIT = System.getProperty("file.separator"); @@ -913,10 +913,19 @@ private void initScopURLs() { // first, try default scop ScopMirror primary = new ScopMirror(); // If unreachable, try alternate Berkeley location - ScopMirror alt = new ScopMirror( - SCOP_DOWNLOAD_ALTERNATE, - "dir.cla.scop.%s.txt","dir.des.scop.%s.txt", - "dir.hie.scop.%s.txt","dir.com.scop.%s.txt"); + ScopMirror alt; + if (scopVersion.startsWith("2.")) { + alt = new ScopMirror( + SCOP_DOWNLOAD_ALTERNATE, + "dir.cla.scope.%s.txt","dir.des.scope.%s.txt", + "dir.hie.scope.%s.txt","dir.com.scope.%s.txt"); + } + else { + alt = new ScopMirror( + SCOP_DOWNLOAD_ALTERNATE, + "dir.cla.scop.%s.txt","dir.des.scop.%s.txt", + "dir.hie.scop.%s.txt","dir.com.scop.%s.txt"); + } mirrors.add(primary); mirrors.add(alt); } diff --git a/biojava-survival/pom.xml b/biojava-survival/pom.xml index 61bcb2369f..c9c0c60a55 100644 --- a/biojava-survival/pom.xml +++ b/biojava-survival/pom.xml @@ -4,7 +4,7 @@ org.biojava biojava - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT biojava-survival diff --git a/biojava-ws/pom.xml b/biojava-ws/pom.xml index 86cf1004f9..258814c27d 100644 --- a/biojava-ws/pom.xml +++ b/biojava-ws/pom.xml @@ -3,7 +3,7 @@ biojava org.biojava - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT biojava-ws biojava-ws @@ -19,7 +19,7 @@ org.biojava biojava-core - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT compile diff --git a/pom.xml b/pom.xml index 1b079ca711..efcddb8feb 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ org.biojava biojava pom - 7.0.3-SNAPSHOT + 7.1.0-SNAPSHOT biojava BioJava is an open-source project dedicated to providing a Java framework for processing biological data. It provides analytical and statistical routines, parsers for common file formats and allows the