diff --git a/model/src/main/java/org/gorpipe/gor/driver/providers/stream/datatypes/cram/CramIterator.java b/model/src/main/java/org/gorpipe/gor/driver/providers/stream/datatypes/cram/CramIterator.java index 174f2dff2..b46c2d495 100644 --- a/model/src/main/java/org/gorpipe/gor/driver/providers/stream/datatypes/cram/CramIterator.java +++ b/model/src/main/java/org/gorpipe/gor/driver/providers/stream/datatypes/cram/CramIterator.java @@ -278,12 +278,14 @@ private CRAMReferenceSource createFileReference(File refFile, boolean preferFold if (refFile.isDirectory()) { return createCompositeReferenceSource(refFile); } else if (preferFolder) { - try { - return createCompositeReferenceSource(refFile.getParentFile()); - } catch (Exception e) { - // Fallback to single file, in case none of the files contains proper meta. - return createSharedFastaReferenceSource(refFile); + var folderSource = new FolderReferenceSource(refFile.getParentFile().getPath()); + if (folderSource.containsReferenceFile(refFile.toPath())) { + return createCompositeReferenceSource(folderSource, refFile.getParentFile()); } + // Fallback to single file, in case the reference file does not contain proper meta. + log.debug("Reference file {} is not md5 indexed, using it as single fasta reference", refFile.getPath()); + folderSource.close(); + return createSharedFastaReferenceSource(refFile); } else { return createSharedFastaReferenceSource(refFile); } @@ -298,9 +300,13 @@ private CRAMReferenceSource createSharedFastaReferenceSource(File refFile) { return new SharedFastaReferenceSource(referenceFile, referenceKey); } private CRAMReferenceSource createCompositeReferenceSource(File refFolder) { + return createCompositeReferenceSource(new FolderReferenceSource(refFolder.getPath()), refFolder); + } + + private CRAMReferenceSource createCompositeReferenceSource(FolderReferenceSource folderSource, File refFolder) { log.debug("Using folder reference for CRAM: {}", refFolder.getPath()); return new CompositeReferenceSource(List.of( - new FolderReferenceSource(refFolder.getPath()), + folderSource, new EBIReferenceSource(refFolder.getPath()))); } } diff --git a/model/src/main/java/org/gorpipe/gor/driver/providers/stream/datatypes/cram/reference/EBIReferenceSource.java b/model/src/main/java/org/gorpipe/gor/driver/providers/stream/datatypes/cram/reference/EBIReferenceSource.java index 85add05a9..d035d21fa 100644 --- a/model/src/main/java/org/gorpipe/gor/driver/providers/stream/datatypes/cram/reference/EBIReferenceSource.java +++ b/model/src/main/java/org/gorpipe/gor/driver/providers/stream/datatypes/cram/reference/EBIReferenceSource.java @@ -14,8 +14,10 @@ import java.io.IOException; import java.io.InputStream; import java.net.URL; +import java.net.URLConnection; import java.nio.file.Files; import java.nio.file.Path; +import java.util.HashMap; import java.util.HashSet; import java.util.Locale; import java.util.Map; @@ -30,15 +32,19 @@ public class EBIReferenceSource extends MD5CachedReferenceSource { private static final Logger log = LoggerFactory.getLogger(EBIReferenceSource.class); public static final String KEY_USE_CRAM_REF_DOWNLOAD = "gor.driver.cram.ref.download"; + public static final String KEY_CRAM_REF_DOWNLOAD_URL_MASK = "gor.driver.cram.ref.download.urlmask"; + public static final String KEY_CRAM_REF_DOWNLOAD_TIMEOUT = "gor.driver.cram.ref.download.timeout"; // Milliseconds private static final String REFBASES_PREFIX = "md5_"; private static final String REFBASES_EXT = ".txt"; - protected static Map md5ToRefbases = new ConcurrentHashMap<>(); + // Shared between instances, as scanning the folder is expensive. + private static final Map> indexByFolder = new ConcurrentHashMap<>(); private static final int DOWNLOAD_TRIES_BEFORE_FAILING = 2; private Path referenceFolder; // If null we do not download. + private Md5FolderIndex md5ToRefbases; // Null if no reference folder. public EBIReferenceSource() { } @@ -50,37 +56,29 @@ public EBIReferenceSource(String referenceFolder) { if (!Files.isDirectory(this.referenceFolder)) { throw new GorResourceException("Can not create FolderReferenceSource %s as the target is not a folder or does not exists".formatted(referenceFolder), referenceFolder); } - scanReferenceFolder(); + md5ToRefbases = Md5FolderIndex.get(indexByFolder, this.referenceFolder, EBIReferenceSource::scanReferenceFolder); + md5ToRefbases.refreshIfStale(); } } Set getRefbasesFiles() { - return new HashSet<>(md5ToRefbases.values()); + return md5ToRefbases != null ? new HashSet<>(md5ToRefbases.values()) : new HashSet<>(); } - private void scanReferenceFolder() { - md5ToRefbases.clear(); - - if (referenceFolder == null) return; - - try { - for (var p : Files.list(referenceFolder).filter(Files::isRegularFile).toList()) { + private static Map scanReferenceFolder(Path referenceFolder) { + Map md5Map = new HashMap<>(); + try (var files = Files.list(referenceFolder)) { + for (var p : files.filter(Files::isRegularFile).toList()) { var f = p.getFileName().toString().toLowerCase(); if (f.startsWith(REFBASES_PREFIX) && f.endsWith(REFBASES_EXT)) { - processRefbasesFile(referenceFolder.resolve(f)); + String md5 = f.substring(REFBASES_PREFIX.length(), f.length() - REFBASES_EXT.length()); + md5Map.put(md5, p); } } } catch (IOException e) { log.warn("Failed scanning reference folder {}", referenceFolder, e); } - } - - private void processRefbasesFile(Path refbases) { - String fileName = refbases.getFileName().toString(); - if (!fileName.startsWith(REFBASES_PREFIX) || !fileName.endsWith(REFBASES_EXT)) return; - - String md5 = fileName.substring(REFBASES_PREFIX.length(), fileName.length() - REFBASES_EXT.length()); - md5ToRefbases.put(md5, refbases); + return md5Map; } @Override @@ -88,7 +86,7 @@ protected byte[] loadReference(final SAMSequenceRecord record) { var md5 = record.getMd5(); // Load from refbases file. - Path refbasesPath = md5ToRefbases.get(md5); + Path refbasesPath = md5ToRefbases != null ? md5ToRefbases.get(md5) : null; if (refbasesPath != null) { try { byte[] bases = Files.readAllBytes(refbasesPath); @@ -125,10 +123,16 @@ protected byte[] loadReference(final SAMSequenceRecord record) { * @throws IOException if the sequence is not found or the download fails. */ private byte[] downloadFromEBI(final String md5) throws IOException { - final String url = String.format(Locale.US, Defaults.EBI_REFERENCE_SERVICE_URL_MASK, md5); + final String urlMask = System.getProperty(KEY_CRAM_REF_DOWNLOAD_URL_MASK, Defaults.EBI_REFERENCE_SERVICE_URL_MASK); + final String url = String.format(Locale.US, urlMask, md5); + // Without timeouts an unresponsive server blocks forever (while holding the reference source lock). + final int timeout = Integer.parseInt(System.getProperty(KEY_CRAM_REF_DOWNLOAD_TIMEOUT, "30000")); for (int i = 0; i < DOWNLOAD_TRIES_BEFORE_FAILING; i++) { - try (final InputStream is = new URL(url).openStream()) { + final URLConnection connection = new URL(url).openConnection(); + connection.setConnectTimeout(timeout); + connection.setReadTimeout(timeout); + try (final InputStream is = connection.getInputStream()) { if (is == null) return null; diff --git a/model/src/main/java/org/gorpipe/gor/driver/providers/stream/datatypes/cram/reference/FolderReferenceSource.java b/model/src/main/java/org/gorpipe/gor/driver/providers/stream/datatypes/cram/reference/FolderReferenceSource.java index 016b2715a..0531b0ff9 100644 --- a/model/src/main/java/org/gorpipe/gor/driver/providers/stream/datatypes/cram/reference/FolderReferenceSource.java +++ b/model/src/main/java/org/gorpipe/gor/driver/providers/stream/datatypes/cram/reference/FolderReferenceSource.java @@ -49,9 +49,12 @@ public class FolderReferenceSource extends MD5CachedReferenceSource { private static final Set FASTA_EXT = Set.of("fa", "fasta"); private record Md5ReferencePath(String md5, Path path, String contig) {} - protected static Map md5ToReferencePath = new ConcurrentHashMap<>(); - private Path referenceFolder; + // Shared between instances, as scanning the folder is expensive. + private static final Map> indexByFolder = new ConcurrentHashMap<>(); + + private final Path referenceFolder; + private final Md5FolderIndex md5ToReferencePath; private final Map refFileByPath = new ConcurrentHashMap<>(); @@ -60,7 +63,8 @@ public FolderReferenceSource(String referenceFolder) { if (!Files.isDirectory(this.referenceFolder)) { throw new GorResourceException("Can not create FolderReferenceSource %s as the target is not a folder or does not exists".formatted(referenceFolder), referenceFolder); } - scanReferenceFolder(); + md5ToReferencePath = Md5FolderIndex.get(indexByFolder, this.referenceFolder, FolderReferenceSource::scanReferenceFolder); + md5ToReferencePath.refreshIfStale(); } @Override @@ -80,31 +84,37 @@ protected byte[] loadReference(final SAMSequenceRecord record) { return null; } - private void scanReferenceFolder() { - md5ToReferencePath.clear(); - - try { - for (var p : Files.list(referenceFolder).filter(Files::isRegularFile).toList()) { + private static Map scanReferenceFolder(Path referenceFolder) { + Map md5Map = new HashMap<>(); + try (var files = Files.list(referenceFolder)) { + for (var p : files.filter(Files::isRegularFile).toList()) { var f = p.getFileName().toString().toLowerCase(); if (FASTA_EXT.stream().anyMatch(ext -> f.endsWith("." + ext))) { - processFasta(referenceFolder.resolve(f)); + // One bad fasta file should not prevent using the others. + try { + processFasta(p, md5Map); + } catch (Exception e) { + log.error("Skipping invalid cram reference file {}", p, e); + } } } } catch (IOException e) { log.warn("Failed scanning reference folder {}", referenceFolder, e); } + return md5Map; } - private void processFasta(Path fastaFile) { - ReferenceSequenceFile refFile = refFileByPath.computeIfAbsent(fastaFile, ReferenceSequenceFileFactory::getReferenceSequenceFile); - SAMSequenceDictionary dictionary = refFile.getSequenceDictionary(); - if (dictionary == null) { - throw new GorResourceException("Fasta file %s is invalid cram reference as it is missing dict file".formatted(fastaFile), fastaFile.toString()); - } - for (SAMSequenceRecord rec : dictionary.getSequences()) { - String md5 = rec.getMd5(); - if (md5 == null || md5.isEmpty()) continue; - md5ToReferencePath.put(md5, new FolderReferenceSource.Md5ReferencePath(md5, fastaFile, rec.getContig())); + private static void processFasta(Path fastaFile, Map md5Map) throws IOException { + try (ReferenceSequenceFile refFile = ReferenceSequenceFileFactory.getReferenceSequenceFile(fastaFile)) { + SAMSequenceDictionary dictionary = refFile.getSequenceDictionary(); + if (dictionary == null) { + throw new GorResourceException("Fasta file %s is invalid cram reference as it is missing dict file".formatted(fastaFile), fastaFile.toString()); + } + for (SAMSequenceRecord rec : dictionary.getSequences()) { + String md5 = rec.getMd5(); + if (md5 == null || md5.isEmpty()) continue; + md5Map.put(md5, new FolderReferenceSource.Md5ReferencePath(md5, fastaFile, rec.getContig())); + } } } @@ -119,7 +129,17 @@ public void close() { refFileByPath.clear(); } + /** + * @return true if the given fasta file is a valid (md5 indexed) reference file in this folder. + */ + public boolean containsReferenceFile(Path fastaFile) { + Path normalized = fastaFile.toAbsolutePath().normalize(); + return getReferenceFiles().stream().anyMatch(p -> p.toAbsolutePath().normalize().equals(normalized)); + } + Set getReferenceFiles() { - return new HashSet<>(refFileByPath.keySet()); + Set files = new HashSet<>(); + md5ToReferencePath.values().forEach(r -> files.add(r.path())); + return files; } } diff --git a/model/src/main/java/org/gorpipe/gor/driver/providers/stream/datatypes/cram/reference/Md5FolderIndex.java b/model/src/main/java/org/gorpipe/gor/driver/providers/stream/datatypes/cram/reference/Md5FolderIndex.java new file mode 100644 index 000000000..53e5b7bb4 --- /dev/null +++ b/model/src/main/java/org/gorpipe/gor/driver/providers/stream/datatypes/cram/reference/Md5FolderIndex.java @@ -0,0 +1,65 @@ +package org.gorpipe.gor.driver.providers.stream.datatypes.cram.reference; + +import java.nio.file.Path; +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +/** + * Index from md5 to reference data for a reference folder. Shared by all reference sources using the same folder, + * as scanning the folder is expensive. + * + * The folder is rescanned when the index is older than the rescan interval. A rescan builds a new map and swaps it in, + * so readers never see a partially built (or cleared) index (see ENGKNOW-3778). + */ +class Md5FolderIndex { + + public static final String KEY_RESCAN_INTERVAL = "gor.driver.cram.reference.rescaninterval"; // Seconds + + private static final long NOT_SCANNED = -1; + + private final Path folder; + private final Function> scanner; + + private volatile Map md5Map = new ConcurrentHashMap<>(); + private long lastScanMillis = NOT_SCANNED; + + /** + * Get the index for the given folder, creating it if needed. + */ + static Md5FolderIndex get(Map> indexByFolder, Path folder, + Function> scanner) { + return indexByFolder.computeIfAbsent(folder.toAbsolutePath().normalize(), + f -> new Md5FolderIndex<>(f, scanner)); + } + + private Md5FolderIndex(Path folder, Function> scanner) { + this.folder = folder; + this.scanner = scanner; + } + + V get(String md5) { + return md5 != null ? md5Map.get(md5) : null; + } + + synchronized void put(String md5, V value) { + md5Map.put(md5, value); + } + + Collection values() { + return md5Map.values(); + } + + /** + * Rescan the folder if it has not been scanned or the index is older than the rescan interval. + */ + synchronized void refreshIfStale() { + long now = System.currentTimeMillis(); + long intervalMillis = Long.parseLong(System.getProperty(KEY_RESCAN_INTERVAL, "300")) * 1000; + if (lastScanMillis == NOT_SCANNED || now - lastScanMillis >= intervalMillis) { + md5Map = new ConcurrentHashMap<>(scanner.apply(folder)); + lastScanMillis = now; + } + } +} diff --git a/model/src/test/java/org/gorpipe/gor/driver/providers/stream/datatypes/cram/reference/UTestEBIReferenceSource.java b/model/src/test/java/org/gorpipe/gor/driver/providers/stream/datatypes/cram/reference/UTestEBIReferenceSource.java index d460ec11b..2257107df 100644 --- a/model/src/test/java/org/gorpipe/gor/driver/providers/stream/datatypes/cram/reference/UTestEBIReferenceSource.java +++ b/model/src/test/java/org/gorpipe/gor/driver/providers/stream/datatypes/cram/reference/UTestEBIReferenceSource.java @@ -6,11 +6,17 @@ import org.junit.*; import org.junit.rules.TemporaryFolder; +import com.sun.net.httpserver.HttpServer; + import java.io.File; import java.io.FileWriter; import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.ServerSocket; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Set; +import java.util.UUID; /** * Tests for FolderReferenceSource class. @@ -94,6 +100,24 @@ public void testLoadFromRefbasesFile() throws Exception { Assert.assertEquals(1, referenceSource.getRefbasesFiles().size()); } + @Test + public void testCreatingAnotherSourceDoesNotBreakExistingSource() throws Exception { + // ENGKNOW-3778: Opening a new reference source (e.g. by another CRAM iterator) must not affect + // lookups in already open reference sources. + System.setProperty(EBIReferenceSource.KEY_USE_CRAM_REF_DOWNLOAD, "false"); + String md5 = "engknow3778_" + UUID.randomUUID(); + createRefbasesFile(md5, "ACGTACGT".getBytes()); + referenceSource = new EBIReferenceSource(testFolder.getAbsolutePath()); + + File otherFolder = workDir.newFolder("other_ref_folder"); + Files.write(otherFolder.toPath().resolve("md5_engknow3778_" + UUID.randomUUID() + ".txt"), "TTTT".getBytes()); + new EBIReferenceSource(otherFolder.getAbsolutePath()).close(); + + SAMSequenceRecord record = new SAMSequenceRecord("chr3", 8).setMd5(md5); + Assert.assertArrayEquals("GTAC".getBytes(), referenceSource.getReferenceBasesByRegion(record, 2, 4)); + Assert.assertEquals(1, referenceSource.getRefbasesFiles().size()); + } + @Test public void testDownloadFromEBIReal() throws Exception { String md5 = "d2ed829b8a1628d16cbeee88e88e39eb"; // hg19 chrM @@ -115,6 +139,54 @@ public void testDownloadFromEBIReal() throws Exception { Assert.assertArrayEquals(bases, Files.readAllBytes(stored)); } + @Test(timeout = 30000) + public void testDownloadFromUnresponsiveServerTimesOut() throws Exception { + // ENGKNOW-3778: A download from a server that never responds must not hang forever. + try (ServerSocket server = new ServerSocket(0)) { + System.setProperty(EBIReferenceSource.KEY_CRAM_REF_DOWNLOAD_URL_MASK, + "http://localhost:" + server.getLocalPort() + "/%s"); + System.setProperty(EBIReferenceSource.KEY_CRAM_REF_DOWNLOAD_TIMEOUT, "500"); + + referenceSource = new EBIReferenceSource(testFolder.getAbsolutePath()); + SAMSequenceRecord record = new SAMSequenceRecord("chr3", 8).setMd5("engknow3778_" + UUID.randomUUID()); + + Assert.assertNull(referenceSource.getReferenceBases(record, false)); + } finally { + System.clearProperty(EBIReferenceSource.KEY_CRAM_REF_DOWNLOAD_URL_MASK); + System.clearProperty(EBIReferenceSource.KEY_CRAM_REF_DOWNLOAD_TIMEOUT); + } + } + + @Test + public void testDownloadedReferenceIsAddedToSharedIndex() throws Exception { + String md5 = "engknow3778_" + UUID.randomUUID(); + HttpServer server = HttpServer.create(new InetSocketAddress("localhost", 0), 0); + server.createContext("/", exchange -> { + byte[] resp = "ACGTACGT".getBytes(); + exchange.sendResponseHeaders(200, resp.length); + try (var os = exchange.getResponseBody()) { + os.write(resp); + } + }); + server.start(); + try { + System.setProperty(EBIReferenceSource.KEY_CRAM_REF_DOWNLOAD_URL_MASK, + "http://localhost:" + server.getAddress().getPort() + "/%s"); + referenceSource = new EBIReferenceSource(testFolder.getAbsolutePath()); + try (EBIReferenceSource otherSource = new EBIReferenceSource(testFolder.getAbsolutePath())) { + Assert.assertEquals(0, otherSource.getRefbasesFiles().size()); + + SAMSequenceRecord record = new SAMSequenceRecord("chr3", 8).setMd5(md5); + Assert.assertArrayEquals("ACGTACGT".getBytes(), referenceSource.getReferenceBases(record, false)); + + Assert.assertEquals(Set.of(testFolder.toPath().resolve("md5_" + md5 + ".txt")), otherSource.getRefbasesFiles()); + } + } finally { + System.clearProperty(EBIReferenceSource.KEY_CRAM_REF_DOWNLOAD_URL_MASK); + server.stop(0); + } + } + /* @Test public void testDownloadFromEBIFake() throws Exception { diff --git a/model/src/test/java/org/gorpipe/gor/driver/providers/stream/datatypes/cram/reference/UTestFolderReferenceSource.java b/model/src/test/java/org/gorpipe/gor/driver/providers/stream/datatypes/cram/reference/UTestFolderReferenceSource.java index a2c4dd86f..758373e83 100644 --- a/model/src/test/java/org/gorpipe/gor/driver/providers/stream/datatypes/cram/reference/UTestFolderReferenceSource.java +++ b/model/src/test/java/org/gorpipe/gor/driver/providers/stream/datatypes/cram/reference/UTestFolderReferenceSource.java @@ -15,6 +15,8 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Set; +import java.util.UUID; /** * Tests for FolderReferenceSource class. @@ -68,15 +70,24 @@ public void testScanFolderWithValidFastaFiles() throws IOException { Assert.assertEquals(1, referenceSource.getReferenceFiles().size()); } - @Test(expected = GorResourceException.class) + @Test public void testScanFolderIgnoresFastaWithoutIndexFiles() throws IOException { // Create FASTA file without index files File fastaFile = new File(testFolder, "no_index.fasta"); try (FileWriter writer = new FileWriter(fastaFile)) { writer.write(">chr1\nACGTACGT\n"); } + String md5 = "engknow3778_" + UUID.randomUUID(); + createFastaFileWithIndexes("good.fasta", "chr2", "TGCA", md5); + // Bad file is skipped (and logged), other files are still loaded. referenceSource = new FolderReferenceSource(testFolder.getAbsolutePath()); + + Assert.assertEquals(Set.of(new File(testFolder, "good.fasta").toPath()), referenceSource.getReferenceFiles()); + Assert.assertArrayEquals("TGCA".getBytes(), referenceSource.getReferenceBases( + new SAMSequenceRecord("chr2", 4).setMd5(md5), false)); + Assert.assertTrue(referenceSource.containsReferenceFile(new File(testFolder, "good.fasta").toPath())); + Assert.assertFalse(referenceSource.containsReferenceFile(fastaFile.toPath())); } @Test @@ -187,7 +198,7 @@ public void testMultipleCloseCalls() throws IOException { referenceSource.close(); } - @Test(expected = GorResourceException.class) + @Test public void testScanFolderWithCorruptedFastaFile() throws IOException { // Create a FASTA file that exists but is corrupted File fastaFile = new File(testFolder, "corrupted.fasta"); @@ -200,9 +211,15 @@ public void testScanFolderWithCorruptedFastaFile() throws IOException { dictFile.createNewFile(); File faiFile = new File(testFolder, "corrupted.fai"); faiFile.createNewFile(); + String md5 = "engknow3778_" + UUID.randomUUID(); + createFastaFileWithIndexes("good.fasta", "chr2", "TGCA", md5); - // Should handle corrupted files gracefully + // Bad file is skipped (and logged), other files are still loaded. referenceSource = new FolderReferenceSource(testFolder.getAbsolutePath()); + + Assert.assertEquals(Set.of(new File(testFolder, "good.fasta").toPath()), referenceSource.getReferenceFiles()); + Assert.assertArrayEquals("TGCA".getBytes(), referenceSource.getReferenceBases( + new SAMSequenceRecord("chr2", 4).setMd5(md5), false)); } @Test @@ -274,6 +291,48 @@ public void testConcurrentAccess() throws IOException, InterruptedException { } } + @Test + public void testCreatingAnotherSourceDoesNotBreakExistingSource() throws IOException { + // ENGKNOW-3778: Opening a new reference source (e.g. by another CRAM iterator) must not affect + // lookups in already open reference sources. + String md5 = "engknow3778_" + UUID.randomUUID(); + createFastaFileWithIndexes("test.fasta", "chr3", "ACGTACGT", md5); + referenceSource = new FolderReferenceSource(testFolder.getAbsolutePath()); + + File otherFolder = workDir.newFolder("other_ref_folder"); + createFastaFileWithIndexes(otherFolder, "other.fasta", "chr1", "TTTT", "engknow3778_" + UUID.randomUUID()); + try (FolderReferenceSource otherSource = new FolderReferenceSource(otherFolder.getAbsolutePath())) { + SAMSequenceRecord record = new SAMSequenceRecord("chr3", 8).setMd5(md5); + Assert.assertArrayEquals("GTAC".getBytes(), referenceSource.getReferenceBasesByRegion(record, 2, 4)); + } + } + + @Test + public void testFolderIndexIsSharedAndRescannedWhenStale() throws IOException { + createFastaFileWithIndexes("ref1.fasta", "chr1", "ACGT", "engknow3778_" + UUID.randomUUID()); + referenceSource = new FolderReferenceSource(testFolder.getAbsolutePath()); + Assert.assertEquals(1, referenceSource.getReferenceFiles().size()); + + // New file is not seen by new sources until the shared index is stale. + String md5 = "engknow3778_" + UUID.randomUUID(); + createFastaFileWithIndexes("ref2.fasta", "chr2", "TGCA", md5); + SAMSequenceRecord record = new SAMSequenceRecord("chr2", 4).setMd5(md5); + try (FolderReferenceSource source = new FolderReferenceSource(testFolder.getAbsolutePath())) { + Assert.assertEquals(1, source.getReferenceFiles().size()); + Assert.assertNull(source.getReferenceBases(record, false)); + } + + System.setProperty(Md5FolderIndex.KEY_RESCAN_INTERVAL, "0"); + try (FolderReferenceSource source = new FolderReferenceSource(testFolder.getAbsolutePath())) { + Assert.assertEquals(2, source.getReferenceFiles().size()); + Assert.assertArrayEquals("TGCA".getBytes(), source.getReferenceBases(record, false)); + // Existing sources share the index. + Assert.assertEquals(2, referenceSource.getReferenceFiles().size()); + } finally { + System.clearProperty(Md5FolderIndex.KEY_RESCAN_INTERVAL); + } + } + /** * Helper method to create a FASTA file with corresponding .dict and .fai files. * Note: This is a simplified version. In a real implementation, you would need @@ -281,8 +340,13 @@ public void testConcurrentAccess() throws IOException, InterruptedException { */ private File createFastaFileWithIndexes(String fileName, String sequenceName, String sequence, String md5) throws IOException { + return createFastaFileWithIndexes(testFolder, fileName, sequenceName, sequence, md5); + } + + private File createFastaFileWithIndexes(File folder, String fileName, String sequenceName, + String sequence, String md5) throws IOException { // Create FASTA file - File fastaFile = new File(testFolder, fileName); + File fastaFile = new File(folder, fileName); var header = ""; try (FileWriter writer = new FileWriter(fastaFile)) { header = ">" + sequenceName; @@ -303,7 +367,7 @@ private File createFastaFileWithIndexes(String fileName, String sequenceName, // Create .dict file (simplified - real dict files have specific format) String baseName = fileName.substring(0, fileName.lastIndexOf('.')); - File dictFile = new File(testFolder, baseName + ".dict"); + File dictFile = new File(folder, baseName + ".dict"); try (FileWriter writer = new FileWriter(dictFile)) { writer.write("@HD\tVN:1.0\n"); writer.write("@SQ\tSN:" + sequenceName + "\tLN:" + sequence.length()); @@ -314,7 +378,7 @@ private File createFastaFileWithIndexes(String fileName, String sequenceName, } // Create .fai file (FASTA index - simplified) - File faiFile = new File(testFolder, fileName + ".fai"); + File faiFile = new File(folder, fileName + ".fai"); try (FileWriter writer = new FileWriter(faiFile)) { // Format: sequence_name, length, offset, linebases, linewidth // This is simplified - real .fai files need proper calculation