Skip to content

Attempt to fix issue #659 - Error parsing FASTA in multiple passes #687

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
josemduarte merged 3 commits into from
Jul 18, 2017
Merged

Attempt to fix issue #659 - Error parsing FASTA in multiple passes #687

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3a1a0a6
Attempt to fix issue #659 - Error parsing FASTA in multiple passes
arojnuckarin Jul 8, 2017
4f301ce
Revert to old behavior where process(int) return null on end-of-file.
arojnuckarin Jul 8, 2017
76e2499
Last travis-CI build got cancelled. Re-push to trigger it again.
arojnuckarin Jul 8, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 16 additions & 23 deletions biojava-core/src/main/java/org/biojava/nbio/core/sequence/io/FastaReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,43 +188,36 @@ public LinkedHashMap<String,S> process(int max) throws IOException {
line = br.readLine();

if (line == null) {


// Fix for #282
if ( sequences.size() == 0 && max != -1) {
return null;
}

//i.e. EOF
String seq = sb.toString();
if ( seq.length() == 0) {
if ( sb.length() == 0 && header.length() != 0 ) {
logger.warn("Can't parse sequence {}. Got sequence of length 0!", sequenceIndex);
logger.warn("header: {}", header);
}
//logger.info("Sequence index=" + sequenceIndex + " " + fileIndex );
try {
@SuppressWarnings("unchecked")
S sequence = (S)sequenceCreator.getSequence(seq, sequenceIndex);
headerParser.parseHeader(header, sequence);
sequences.put(sequence.getAccession().getID(),sequence);
processedSequences++;
} catch (CompoundNotFoundException e) {
logger.warn("Sequence with header '{}' has unrecognised compounds ({}), it will be ignored",
header, e.getMessage());
header = null;
} else if ( sb.length() > 0 ) {
//logger.info("Sequence index=" + sequenceIndex + " " + fileIndex );
try {
@SuppressWarnings("unchecked")
S sequence = (S)sequenceCreator.getSequence(sb.toString(), sequenceIndex);
headerParser.parseHeader(header, sequence);
sequences.put(sequence.getAccession().getID(),sequence);
processedSequences++;
header = null;
} catch (CompoundNotFoundException e) {
logger.warn("Sequence with header '{}' has unrecognised compounds ({}), it will be ignored",
header, e.getMessage());
}
}
keepGoing = false;
}
if (max > -1 && processedSequences>=max) {
keepGoing=false;
}
if ( this.line == null)
keepGoing = false;
} while (keepGoing);

this.line = line;
this.header= header;

return sequences;
return max > -1 && sequences.isEmpty() ? null : sequences;
}

public void close() throws IOException {
Expand Down
148 changes: 148 additions & 0 deletions biojava-core/src/test/java/org/biojava/nbio/core/fasta/TestFASTAReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
package org.biojava.nbio.core.fasta;

import java.io.InputStream;
import java.util.LinkedHashMap;

import static org.junit.Assert.* ;
import static org.hamcrest.CoreMatchers.* ;

import org.biojava.nbio.core.sequence.ProteinSequence;
import org.biojava.nbio.core.sequence.compound.AminoAcidCompound;
import org.biojava.nbio.core.sequence.compound.AminoAcidCompoundSet;
import org.biojava.nbio.core.sequence.io.FastaReader;
import org.biojava.nbio.core.sequence.io.GenericFastaHeaderParser;
import org.biojava.nbio.core.sequence.io.ProteinSequenceCreator;
import org.biojava.nbio.core.sequence.io.util.ClasspathResource;
import org.junit.Test;


public class TestFASTAReader {

private void testProcessAll(String path) throws Exception {
ClasspathResource r = new ClasspathResource(path);
FastaReader<ProteinSequence, AminoAcidCompound> fastaReader = null ;
try( InputStream inStream = r.getInputStream() ) {
fastaReader = new FastaReader<ProteinSequence, AminoAcidCompound>(
inStream,
new GenericFastaHeaderParser<ProteinSequence, AminoAcidCompound>(),
new ProteinSequenceCreator(AminoAcidCompoundSet.getAminoAcidCompoundSet()));
LinkedHashMap<String, ProteinSequence> sequences = fastaReader.process();
assertThat(sequences,is(notNullValue()));
assertThat(sequences.size(),is(1));
assertThat(sequences.containsKey("P02768"),is(true));
assertThat(sequences.get("P02768").getLength(),is(609));
} finally {
if(fastaReader != null) fastaReader.close();
}
}

/**
* Test file contains one sequence (P02768 from swissprot). Read the whole
* file all at once by calling {@link FastaReader#process()} and verify that
* one sequence is read.
*
* @throws Exception
*/
@Test
public void testProcessAll() throws Exception {
testProcessAll("org/biojava/nbio/core/fasta/P02768.fasta");
}

/**
* Same as {@link #testProcessAll()} but input files contains blank lines
*
* @throws Exception
*/
@Test
public void testProcessAllWithBlankLines() throws Exception {
testProcessAll("org/biojava/nbio/core/fasta/P02768_blank_lines.fasta");
}

private void testProcess1(String path) throws Exception {
ClasspathResource r = new ClasspathResource(path);
FastaReader<ProteinSequence, AminoAcidCompound> fastaReader = null ;
try( InputStream inStream = r.getInputStream() ) {
fastaReader = new FastaReader<ProteinSequence, AminoAcidCompound>(
inStream,
new GenericFastaHeaderParser<ProteinSequence, AminoAcidCompound>(),
new ProteinSequenceCreator(AminoAcidCompoundSet.getAminoAcidCompoundSet()));
LinkedHashMap<String,ProteinSequence> out1 = fastaReader.process(1);
assertThat(out1,is(notNullValue()));
assertThat(out1.size(),is(1));
assertThat(out1.containsKey("P02768"),is(true));
assertThat(out1.get("P02768").getLength(),is(609));
LinkedHashMap<String,ProteinSequence> out2 = fastaReader.process(1);
assertThat(out2,is(nullValue()));
} finally {
if(fastaReader != null) fastaReader.close();
}
}

/**
* Test file contains one sequence (P02768 from swissprot). Read one
* sequence at a time by calling {@link FastaReader#process(int)} and verify
* that the first call get one sequence and the second call get none.
*
* @throws Exception
*/
@Test
public void testProcess1() throws Exception {
testProcess1("org/biojava/nbio/core/fasta/P02768.fasta");
}

/**
* Same as {@link #testProcess1()}, but input contains blank lines.
*
* @throws Exception
*/
@Test
public void testProcess1WithBlankLines() throws Exception {
testProcess1("org/biojava/nbio/core/fasta/P02768_blank_lines.fasta");
}

private void testProcess2(String path) throws Exception {
ClasspathResource r = new ClasspathResource(path);
FastaReader<ProteinSequence, AminoAcidCompound> fastaReader = null ;
try( InputStream inStream = r.getInputStream() ) {
fastaReader = new FastaReader<ProteinSequence, AminoAcidCompound>(
inStream,
new GenericFastaHeaderParser<ProteinSequence, AminoAcidCompound>(),
new ProteinSequenceCreator(AminoAcidCompoundSet.getAminoAcidCompoundSet()));
LinkedHashMap<String,ProteinSequence> out1 = fastaReader.process(1);
assertThat(out1,is(notNullValue()));
assertThat(out1.size(),is(1));
assertThat(out1.containsKey("P02768"),is(true));
assertThat(out1.get("P02768").getLength(),is(609));
LinkedHashMap<String,ProteinSequence> out2 = fastaReader.process(1);
assertThat(out2,is(notNullValue()));
assertThat(out2.size(),is(1));
assertThat(out2.containsKey("P00698"),is(true));
assertThat(out2.get("P00698").getLength(),is(147));
LinkedHashMap<String,ProteinSequence> out3 = fastaReader.process(1);
assertThat(out3,is(nullValue()));
} finally {
if(fastaReader != null) fastaReader.close();
}
}

/**
* Test file contains two sequences. Read one sequence at a time by calling
* {@link FastaReader#process(int)} and verify that the first and second
* call get one sequence each and the third call get none.
*
* @throws Exception
*/
@Test
public void testProcess2() throws Exception {
testProcess2("org/biojava/nbio/core/fasta/TwoSequences.fasta");
}

/**
* Sane as {@link #testProcess2()} but input file contain blank lines
* @throws Exception
*/
@Test
public void testProcess2WithBlankLines() throws Exception {
testProcess2("org/biojava/nbio/core/fasta/TwoSequences_blank_lines.fasta");
}
}
12 changes: 12 additions & 0 deletions biojava-core/src/test/resources/org/biojava/nbio/core/fasta/P02768.fasta
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
>sp|P02768|ALBU_HUMAN Serum albumin OS=Homo sapiens GN=ALB PE=1 SV=2
MKWVTFISLLFLFSSAYSRGVFRRDAHKSEVAHRFKDLGEENFKALVLIAFAQYLQQCPF
EDHVKLVNEVTEFAKTCVADESAENCDKSLHTLFGDKLCTVATLRETYGEMADCCAKQEP
ERNECFLQHKDDNPNLPRLVRPEVDVMCTAFHDNEETFLKKYLYEIARRHPYFYAPELLF
FAKRYKAAFTECCQAADKAACLLPKLDELRDEGKASSAKQRLKCASLQKFGERAFKAWAV
ARLSQRFPKAEFAEVSKLVTDLTKVHTECCHGDLLECADDRADLAKYICENQDSISSKLK
ECCEKPLLEKSHCIAEVENDEMPADLPSLAADFVESKDVCKNYAEAKDVFLGMFLYEYAR
RHPDYSVVLLLRLAKTYETTLEKCCAAADPHECYAKVFDEFKPLVEEPQNLIKQNCELFE
QLGEYKFQNALLVRYTKKVPQVSTPTLVEVSRNLGKVGSKCCKHPEAKRMPCAEDYLSVV
LNQLCVLHEKTPVSDRVTKCCTESLVNRRPCFSALEVDETYVPKEFNAETFTFHADICTL
SEKERQIKKQTALVELVKHKPKATKEQLKAVMDDFAAFVEKCCKADDKETCFAEEGKKLV
AASQAALGL
21 changes: 21 additions & 0 deletions biojava-core/src/test/resources/org/biojava/nbio/core/fasta/P02768_blank_lines.fasta
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
>sp|P02768|ALBU_HUMAN Serum albumin OS=Homo sapiens GN=ALB PE=1 SV=2
MKWVTFISLLFLFSSAYSRGVFRRDAHKSEVAHRFKDLGEENFKALVLIAFAQYLQQCPF
EDHVKLVNEVTEFAKTCVADESAENCDKSLHTLFGDKLCTVATLRETYGEMADCCAKQEP
ERNECFLQHKDDNPNLPRLVRPEVDVMCTAFHDNEETFLKKYLYEIARRHPYFYAPELLF
FAKRYKAAFTECCQAADKAACLLPKLDELRDEGKASSAKQRLKCASLQKFGERAFKAWAV



ARLSQRFPKAEFAEVSKLVTDLTKVHTECCHGDLLECADDRADLAKYICENQDSISSKLK
ECCEKPLLEKSHCIAEVENDEMPADLPSLAADFVESKDVCKNYAEAKDVFLGMFLYEYAR
RHPDYSVVLLLRLAKTYETTLEKCCAAADPHECYAKVFDEFKPLVEEPQNLIKQNCELFE
QLGEYKFQNALLVRYTKKVPQVSTPTLVEVSRNLGKVGSKCCKHPEAKRMPCAEDYLSVV
LNQLCVLHEKTPVSDRVTKCCTESLVNRRPCFSALEVDETYVPKEFNAETFTFHADICTL
SEKERQIKKQTALVELVKHKPKATKEQLKAVMDDFAAFVEKCCKADDKETCFAEEGKKLV
AASQAALGL






16 changes: 16 additions & 0 deletions biojava-core/src/test/resources/org/biojava/nbio/core/fasta/TwoSequences.fasta
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
>sp|P02768|ALBU_HUMAN Serum albumin OS=Homo sapiens GN=ALB PE=1 SV=2
MKWVTFISLLFLFSSAYSRGVFRRDAHKSEVAHRFKDLGEENFKALVLIAFAQYLQQCPF
EDHVKLVNEVTEFAKTCVADESAENCDKSLHTLFGDKLCTVATLRETYGEMADCCAKQEP
ERNECFLQHKDDNPNLPRLVRPEVDVMCTAFHDNEETFLKKYLYEIARRHPYFYAPELLF
FAKRYKAAFTECCQAADKAACLLPKLDELRDEGKASSAKQRLKCASLQKFGERAFKAWAV
ARLSQRFPKAEFAEVSKLVTDLTKVHTECCHGDLLECADDRADLAKYICENQDSISSKLK
ECCEKPLLEKSHCIAEVENDEMPADLPSLAADFVESKDVCKNYAEAKDVFLGMFLYEYAR
RHPDYSVVLLLRLAKTYETTLEKCCAAADPHECYAKVFDEFKPLVEEPQNLIKQNCELFE
QLGEYKFQNALLVRYTKKVPQVSTPTLVEVSRNLGKVGSKCCKHPEAKRMPCAEDYLSVV
LNQLCVLHEKTPVSDRVTKCCTESLVNRRPCFSALEVDETYVPKEFNAETFTFHADICTL
SEKERQIKKQTALVELVKHKPKATKEQLKAVMDDFAAFVEKCCKADDKETCFAEEGKKLV
AASQAALGL
>sp|P00698|LYSC_CHICK Lysozyme C OS=Gallus gallus GN=LYZ PE=1 SV=1
MRSLLILVLCFLPLAALGKVFGRCELAAAMKRHGLDNYRGYSLGNWVCAAKFESNFNTQA
TNRNTDGSTDYGILQINSRWWCNDGRTPGSRNLCNIPCSALLSSDITASVNCAKKIVSDG
NGMNAWVAWRNRCKGTDVQAWIRGCRL
21 changes: 21 additions & 0 deletions biojava-core/src/test/resources/org/biojava/nbio/core/fasta/TwoSequences_blank_lines.fasta
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
>sp|P02768|ALBU_HUMAN Serum albumin OS=Homo sapiens GN=ALB PE=1 SV=2
MKWVTFISLLFLFSSAYSRGVFRRDAHKSEVAHRFKDLGEENFKALVLIAFAQYLQQCPF
EDHVKLVNEVTEFAKTCVADESAENCDKSLHTLFGDKLCTVATLRETYGEMADCCAKQEP
ERNECFLQHKDDNPNLPRLVRPEVDVMCTAFHDNEETFLKKYLYEIARRHPYFYAPELLF


FAKRYKAAFTECCQAADKAACLLPKLDELRDEGKASSAKQRLKCASLQKFGERAFKAWAV
ARLSQRFPKAEFAEVSKLVTDLTKVHTECCHGDLLECADDRADLAKYICENQDSISSKLK
ECCEKPLLEKSHCIAEVENDEMPADLPSLAADFVESKDVCKNYAEAKDVFLGMFLYEYAR
RHPDYSVVLLLRLAKTYETTLEKCCAAADPHECYAKVFDEFKPLVEEPQNLIKQNCELFE
QLGEYKFQNALLVRYTKKVPQVSTPTLVEVSRNLGKVGSKCCKHPEAKRMPCAEDYLSVV
LNQLCVLHEKTPVSDRVTKCCTESLVNRRPCFSALEVDETYVPKEFNAETFTFHADICTL
SEKERQIKKQTALVELVKHKPKATKEQLKAVMDDFAAFVEKCCKADDKETCFAEEGKKLV
AASQAALGL


>sp|P00698|LYSC_CHICK Lysozyme C OS=Gallus gallus GN=LYZ PE=1 SV=1
MRSLLILVLCFLPLAALGKVFGRCELAAAMKRHGLDNYRGYSLGNWVCAAKFESNFNTQA

TNRNTDGSTDYGILQINSRWWCNDGRTPGSRNLCNIPCSALLSSDITASVNCAKKIVSDG
NGMNAWVAWRNRCKGTDVQAWIRGCRL