Skip to content

Commit 92a7ec0

Browse files
josemduartejosemduarte
authored
Merge pull request #1134 from aalhossary/aa/electron-density-maps
Fetch, cache and display electron density and cryo-EM maps
2 parents e84025e + d5a8f1f commit 92a7ec0

34 files changed

Lines changed: 5512 additions & 0 deletions

‎biojava-integrationtest/src/test/java/org/biojava/nbio/structure/test/io/density/DensityMapIntegrationTest.java‎

Lines changed: 215 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,215 @@
1+
/**
2+
* BioJava development code
3+
*
4+
* This code may be freely distributed and modified under the terms of the GNU
5+
* Lesser General Public Licence. This should be distributed with the code. If
6+
* you do not have a copy, see:
7+
*
8+
* http://www.gnu.org/copyleft/lesser.html
9+
*
10+
* Copyright for this code is held jointly by the individual authors. These
11+
* should be listed in @author doc comments.
12+
*
13+
* For more information on the BioJava project and its aims, or to join the
14+
* biojava-l mailing list, visit the home page at:
15+
*
16+
* http://www.biojava.org/
17+
*/
18+
package org.biojava.nbio.structure.test.io.density;
19+
20+
import static org.junit.jupiter.api.Assertions.assertEquals;
21+
import static org.junit.jupiter.api.Assertions.assertFalse;
22+
import static org.junit.jupiter.api.Assertions.assertNotNull;
23+
import static org.junit.jupiter.api.Assertions.assertTrue;
24+
import static org.junit.jupiter.api.Assertions.fail;
25+
26+
import java.io.File;
27+
import java.io.IOException;
28+
import java.nio.charset.StandardCharsets;
29+
import java.nio.file.Files;
30+
import java.util.Arrays;
31+
import java.util.List;
32+
33+
import org.biojava.nbio.core.util.FileDownloadUtils;
34+
import org.biojava.nbio.structure.PdbId;
35+
import org.biojava.nbio.structure.io.density.Ccp4Header;
36+
import org.biojava.nbio.structure.io.density.DensityFileFormat;
37+
import org.biojava.nbio.structure.io.density.DensityMapCache;
38+
import org.biojava.nbio.structure.io.density.DensityMapKind;
39+
import org.biojava.nbio.structure.io.density.DensityMapRequest;
40+
import org.biojava.nbio.structure.io.density.DensityMapResult;
41+
import org.biojava.nbio.structure.io.density.DensityMapSource;
42+
import org.biojava.nbio.structure.io.density.NoDensityMapException;
43+
import org.junit.jupiter.api.AfterEach;
44+
import org.junit.jupiter.api.BeforeEach;
45+
import org.junit.jupiter.api.Test;
46+
47+
/**
48+
* Density fetching against the real services.
49+
* <p>
50+
* Deliberately frugal: the entries chosen keep a full run to a couple of
51+
* megabytes plus a few small metadata calls. In particular the cryo-EM path is
52+
* exercised with the size limit set so low that the 116 MB map is declined
53+
* before any of its body is transferred, which tests the whole resolution and
54+
* guard sequence without the download.
55+
*
56+
* @author Amr ALHOSSARY
57+
* @since 7.3.0
58+
*/
59+
public class DensityMapIntegrationTest {
60+
61+
private File cacheRoot;
62+
private DensityMapCache cache;
63+
64+
@BeforeEach
65+
public void setUp() throws IOException {
66+
cacheRoot = Files.createTempDirectory("bj-density-it").toFile();
67+
cache = new DensityMapCache(cacheRoot.getAbsolutePath());
68+
}
69+
70+
@AfterEach
71+
public void tearDown() throws IOException {
72+
FileDownloadUtils.deleteDirectory(cacheRoot.toPath());
73+
}
74+
75+
/** The default path for an X-ray entry: the smallest source answers first. */
76+
@Test
77+
public void fetchesAnXrayMapFromTheFirstSourceTried() throws IOException {
78+
DensityMapResult result = cache.getDensityMap(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC);
79+
80+
assertEquals(DensityMapSource.RCSB_VOLUME_SERVER, result.getSource());
81+
assertEquals(DensityMapKind.TWO_FO_FC, result.getKind());
82+
assertTrue(result.isRenderable());
83+
assertFalse(result.isFromCache());
84+
assertTrue(result.getFileSizeBytes() > 1024);
85+
assertTrue(DensityMapResult.metaFileFor(result.getFile()).isFile(),
86+
"a .meta sidecar makes the result reconstructible offline");
87+
88+
// second call must come from the cache without another download
89+
DensityMapResult again = cache.getDensityMap(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC);
90+
assertTrue(again.isFromCache());
91+
assertEquals(result.getFile(), again.getFile());
92+
}
93+
94+
/**
95+
* Both map kinds come out of one download, and the difference map is presented
96+
* under the companion name that makes Jmol read the other data block.
97+
*/
98+
@Test
99+
public void bothKindsShareASingleDownload() throws IOException {
100+
DensityMapResult twoFoFc = cache.getDensityMap(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC);
101+
DensityMapResult foFc = cache.getDensityMap(new PdbId("1cbs"), DensityMapKind.FO_FC);
102+
103+
assertEquals(DensityMapKind.FO_FC, foFc.getKind());
104+
assertFalse(twoFoFc.getFile().equals(foFc.getFile()), "the difference map needs its own file name");
105+
assertTrue(foFc.getFile().getName().contains("&diff=1"),
106+
"the marker has to be in the name for Jmol to select the FO-FC block");
107+
assertEquals(twoFoFc.getFileSizeBytes(), foFc.getFileSizeBytes(),
108+
"both names must address the same bytes");
109+
}
110+
111+
/** PDBe serves real CCP4 files, which the header check should recognise. */
112+
@Test
113+
public void pdbeServesAGenuineCcp4Map() throws IOException {
114+
cache.setSourceChain(DensityMapKind.TWO_FO_FC, Arrays.asList(DensityMapSource.PDBE_CCP4));
115+
DensityMapResult result = cache.getDensityMap(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC);
116+
117+
assertEquals(DensityMapSource.PDBE_CCP4, result.getSource());
118+
assertEquals(DensityFileFormat.CCP4, result.getFormat());
119+
assertTrue(Ccp4Header.isCcp4(result.getFile()), "the CCP4 stamp should be present at byte 208");
120+
assertTrue(FileDownloadUtils.validateFile(result.getFile()));
121+
}
122+
123+
/**
124+
* The whole cryo-EM route: resolve the EMDB entry, pick up the author contour
125+
* level, and decline the full map on size without transferring it.
126+
*/
127+
@Test
128+
public void resolvesCryoEmEntriesAndHonoursTheSizeLimit() throws IOException {
129+
List<String> emdbIds = cache.getEmdbResolver().getEmdbIds(new PdbId("6hu9"));
130+
assertEquals(Arrays.asList("EMD-0262"), emdbIds);
131+
132+
DensityMapResult result = cache.getDensityMap(new PdbId("6hu9"), DensityMapKind.AUTO);
133+
assertEquals(DensityMapKind.EM, result.getKind());
134+
assertEquals("EMD-0262", result.getEmdbId());
135+
assertNotNull(result.getRecommendedContourLevel(),
136+
"EM maps need the author contour level to be displayed properly");
137+
assertEquals(0.0263, result.getRecommendedContourLevel(), 1e-6);
138+
assertNotNull(result.getContourInSigma());
139+
140+
// With only the full archive enabled and a tiny ceiling, the guard must fire
141+
// rather than pulling down 116 MB.
142+
DensityMapCache strict = new DensityMapCache(cacheRoot.getAbsolutePath());
143+
strict.setSourceChain(DensityMapKind.EM, Arrays.asList(DensityMapSource.EMDB_MAP));
144+
strict.setMaxDownloadBytes(1024);
145+
try {
146+
strict.getDensityMap(DensityMapRequest.builder(new PdbId("6hu9")).kind(DensityMapKind.EM).build());
147+
fail("the size guard should have declined the full EMDB map");
148+
} catch (NoDensityMapException e) {
149+
assertTrue(e.getAttempts().get(DensityMapSource.EMDB_MAP).contains("too large"));
150+
}
151+
}
152+
153+
/** 4HHB was deposited in 1984 without structure factors, so nothing has a map for it. */
154+
@Test
155+
public void reportsWhyAnEntryHasNoDensity() throws IOException {
156+
cache.setSourceEnabled(DensityMapSource.WWPDB_MAP_COEFFICIENTS, true);
157+
try {
158+
cache.getDensityMap(new PdbId("4hhb"), DensityMapKind.AUTO);
159+
fail("4hhb has no deposited structure factors");
160+
} catch (NoDensityMapException e) {
161+
assertFalse(e.getAttempts().isEmpty());
162+
assertTrue(e.getAttempts().values().stream().anyMatch(r -> r.contains("404")));
163+
}
164+
}
165+
166+
/**
167+
* Coefficients must arrive intact and be verifiable afterwards. Whether that
168+
* verification includes a cryptographic digest depends on the server.
169+
* <p>
170+
* The divided archive paths on files.wwpdb.org and files.rcsb.org return the content
171+
* MD5 as the ETag. The flat /validation/download/ endpoint this provider now uses
172+
* returns neither an ETag nor a Content-Length, and neither does the beta archive on
173+
* those two hosts, so no digest can be recorded there. The size sidecar is written
174+
* from the bytes actually read, so it exists either way.
175+
* <p>
176+
* The digest is therefore asserted when the server offered one and skipped when it
177+
* did not, rather than being required: requiring it would fail against the endpoint
178+
* we use, and hard-coding the divided path would only work until the archive
179+
* transition in July 2027.
180+
*/
181+
@Test
182+
public void mapCoefficientsArriveIntactAndVerifiable() throws IOException {
183+
cache.setSourceEnabled(DensityMapSource.WWPDB_MAP_COEFFICIENTS, true);
184+
cache.setSourceChain(DensityMapKind.TWO_FO_FC, Arrays.asList(DensityMapSource.WWPDB_MAP_COEFFICIENTS));
185+
186+
DensityMapResult result = cache.getDensityMap(DensityMapRequest.builder(new PdbId("1cbs"))
187+
.kind(DensityMapKind.TWO_FO_FC)
188+
.allowNonRenderableFormats(true)
189+
.build());
190+
191+
assertEquals(DensityMapSource.WWPDB_MAP_COEFFICIENTS, result.getSource());
192+
assertFalse(result.isRenderable(),
193+
"structure factors are not a map and must not claim to be renderable");
194+
195+
// written from the observed byte count, so it is present whether or not the
196+
// server declared a length
197+
assertTrue(FileDownloadUtils.validateFile(result.getFile()),
198+
"a freshly downloaded file must validate against its own sidecars");
199+
200+
File hashFile = new File(result.getFile().getParentFile(), result.getFile().getName() + ".hash_MD5");
201+
if (hashFile.isFile()) {
202+
String recorded = new String(Files.readAllBytes(hashFile.toPath()), StandardCharsets.UTF_8).trim();
203+
assertTrue(FileDownloadUtils.verifyHash(result.getFile(), FileDownloadUtils.Hash.MD5, recorded),
204+
"the recorded MD5 must match the file it describes");
205+
} else {
206+
System.out.println("No MD5 recorded for " + result.getSourceUrl()
207+
+ " - the server offered no usable ETag. Size validation still applies.");
208+
}
209+
210+
// corrupt it and confirm validation actually catches it
211+
Files.write(result.getFile().toPath(), new byte[] {0, 1, 2, 3});
212+
assertFalse(FileDownloadUtils.validateFile(result.getFile()),
213+
"a truncated file must not validate");
214+
}
215+
}

‎biojava-structure-gui/src/main/java/demo/DemoShowElectronDensity.java‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
/**
2+
* BioJava development code
3+
*
4+
* This code may be freely distributed and modified under the terms of the GNU
5+
* Lesser General Public Licence. This should be distributed with the code. If
6+
* you do not have a copy, see:
7+
*
8+
* http://www.gnu.org/copyleft/lesser.html
9+
*
10+
* Copyright for this code is held jointly by the individual authors. These
11+
* should be listed in @author doc comments.
12+
*
13+
* For more information on the BioJava project and its aims, or to join the
14+
* biojava-l mailing list, visit the home page at:
15+
*
16+
* http://www.biojava.org/
17+
*/
18+
package demo;
19+
20+
import org.biojava.nbio.structure.PdbId;
21+
import org.biojava.nbio.structure.Structure;
22+
import org.biojava.nbio.structure.StructureIO;
23+
import org.biojava.nbio.structure.align.gui.jmol.StructureAlignmentJmol;
24+
import org.biojava.nbio.structure.io.density.DensityMapCache;
25+
import org.biojava.nbio.structure.io.density.DensityMapKind;
26+
import org.biojava.nbio.structure.io.density.DensityMapResult;
27+
28+
/**
29+
* Shows 1CBS with its electron density drawn around the bound retinoic acid.
30+
* <p>
31+
* Both maps are displayed: the 2mFo-DFc map in blue at 1 sigma, which should hug
32+
* the ligand closely, and the mFo-DFc difference map as a red and green pair at
33+
* 3 sigma, which for a well refined structure should show very little.
34+
* <p>
35+
* Once the window is up, the map can be manipulated from the Rasmol command box
36+
* at the bottom, for instance
37+
* <pre>
38+
* isosurface ID "bj_density_2fofc" delete
39+
* </pre>
40+
* to remove just the blue surface. Pressing Reset Display keeps the maps, since
41+
* they are folded into the saved state when they are drawn.
42+
*
43+
* @author Amr ALHOSSARY
44+
* @since 7.3.0
45+
*/
46+
public class DemoShowElectronDensity {
47+
48+
/**
49+
* @param args an optional PDB ID to display instead of the default
50+
* @throws Exception if the structure or the map could not be fetched
51+
*/
52+
public static void main(String[] args) throws Exception {
53+
String id = args.length > 0 ? args[0] : "1cbs";
54+
55+
Structure structure = StructureIO.getStructure(id);
56+
StructureAlignmentJmol viewer = new StructureAlignmentJmol();
57+
viewer.setStructure(structure);
58+
viewer.evalString("select all; cartoon on; color chain; "
59+
+ "select ligand; wireframe 0.16; spacefill 0.4; color cpk;");
60+
61+
DensityMapCache cache = new DensityMapCache();
62+
System.out.println("Density cache: " + cache.getCachePath());
63+
64+
// Clipping to the ligand keeps the surface readable and, for a large map,
65+
// keeps the contouring quick enough not to stall the interface.
66+
for (DensityMapKind kind : new DensityMapKind[] {DensityMapKind.TWO_FO_FC, DensityMapKind.FO_FC}) {
67+
cache.findDensityMap(new PdbId(id), kind).ifPresent(map -> {
68+
System.out.printf("%-8s from %-20s %s (%,d bytes)%n",
69+
map.getKind(), map.getSource(), map.getFile().getName(), map.getFileSizeBytes());
70+
viewer.getJmolPanel().loadDensityMap(map, "{ligand}", 5.0);
71+
});
72+
}
73+
74+
DensityMapResult any = cache.findDensityMap(new PdbId(id), DensityMapKind.AUTO).orElse(null);
75+
if (any == null) {
76+
System.out.println("No density is available for " + id
77+
+ " - try an entry with deposited structure factors, such as 1cbs.");
78+
}
79+
}
80+
}

‎biojava-structure-gui/src/main/java/org/biojava/nbio/structure/align/gui/MenuCreator.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,8 @@ public class MenuCreator {
6666
public static final String PAIRWISE_ALIGN = "New Pairwise Alignment";
6767
public static final String MULTIPLE_ALIGN = "New Multiple Alignment";
6868
public static final String PHYLOGENETIC_TREE = "Phylogenetic Tree";
69+
/** @since 7.3.0 */
70+
public static final String SHOW_DENSITY = "Show Electron Density";
6971
protected static final int keyMask =
7072
Toolkit.getDefaultToolkit().getMenuShortcutKeyMask();
7173

@@ -169,6 +171,8 @@ public static JMenuBar initJmolMenu(JFrame frame,
169171
distMax.setMnemonic(KeyEvent.VK_D);
170172
distMax.addActionListener(new MyDistMaxListener(parent));
171173
view.add(distMax);
174+
//Electron density
175+
view.add(getShowDensityMenuItem(parent));
172176
//Dot Plot - only if the alignment was an afpChain
173177
if (afpChain != null){
174178
JMenuItem dotplot = new JMenuItem(DOT_PLOT);
@@ -229,6 +233,22 @@ public static JMenuItem getOpenPDBMenuItem() {
229233
return openI;
230234
}
231235

236+
/**
237+
* Menu item that fetches and displays the electron density or cryo-EM map for
238+
* the structure currently on screen.
239+
*
240+
* @param parent the viewer to draw the map into
241+
* @return the menu item
242+
* @author Amr ALHOSSARY
243+
* @since 7.3.0
244+
*/
245+
public static JMenuItem getShowDensityMenuItem(AbstractAlignmentJmol parent) {
246+
JMenuItem densityI = new JMenuItem(SHOW_DENSITY);
247+
densityI.setMnemonic(KeyEvent.VK_E);
248+
densityI.addActionListener(new MyShowDensityListener(parent));
249+
return densityI;
250+
}
251+
232252

233253
public static JMenuItem getLoadMenuItem() {
234254

0 commit comments

Comments
 (0)