Begin implementing Opus metadata provider

src/main/java/org/deepsymmetry/beatlink/data/CrateDigger.java

@@ -252,7 +252,6 @@ private void fetchFile(SlotReference slot, String path, File destination) throws
      * @throws IOException if there is a problem fetching the file
      */
     private void fetchFile(SlotReference slot, String path, File destination, int retryLimit) throws IOException {
-        destination.deleteOnExit();
         final DeviceAnnouncement player = DeviceFinder.getInstance().getLatestAnnouncementFrom(slot.player);
         if (player == null) {
             throw new IOException("Cannot fetch file from player that is not found on the network; slot: " + slot);
@@ -264,6 +263,7 @@ private void fetchFile(SlotReference slot, String path, File destination, int re
         while (triesMade < retryLimit) {
             try {
                 FileFetcher.getInstance().fetch(player.getAddress(), mountPath(slot.slot), path, destination);
+                destination.deleteOnExit();
                 return;
             } catch (IOException e) {
                 if (path.startsWith("PIONEER/") &&
@@ -738,7 +738,7 @@ public static CrateDigger getInstance() {
      *
      * @return the created temporary directory.
      */
-    private File createDownloadDirectory() {
+    static File createDownloadDirectory() {
         File baseDir = new File(System.getProperty("java.io.tmpdir"));
         String baseName = "bl-" + System.currentTimeMillis() + "-";
 

src/main/java/org/deepsymmetry/beatlink/data/OpusProvider.java

@@ -1,12 +1,20 @@
 package org.deepsymmetry.beatlink.data;
 
+import io.kaitai.struct.RandomAccessFileKaitaiStream;
+import org.deepsymmetry.beatlink.CdjStatus;
 import org.deepsymmetry.beatlink.MediaDetails;
+import org.deepsymmetry.beatlink.Util;
 import org.deepsymmetry.cratedigger.Database;
 import org.deepsymmetry.cratedigger.pdb.RekordboxAnlz;
+import org.deepsymmetry.cratedigger.pdb.RekordboxPdb;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.File;
+import java.io.IOException;
 import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -62,18 +70,271 @@ public boolean isRunning() {
      */
     private final AtomicReference<Database> usb2database = new AtomicReference<Database>();
 
+    /**
+     * Attach a metadata archive to supply information for the media mounted a USB slot of the Opus Quad.
+     * This must be a file created using {@link org.deepsymmetry.cratedigger.Archivist#createArchive(Database, File)}
+     * from that media.
+     *
+     * @param archive the metadata archive that can provide metadata for tracks playing from the specified USB slot, or
+     *                {@code null} to stop providing metadata for that slot
+     * @param slot which USB slot we are to provide metadata for (we follow the XDJ-XZ convention of using
+     *            {@link CdjStatus.TrackSourceSlot#SD_SLOT} to represent USB 1, and
+     *            {@link CdjStatus.TrackSourceSlot#USB_SLOT} to represent USB 2)
+     *
+     * @throws java.io.IOException if there is a problem attaching the archive
+     * @throws IllegalArgumentException if a slot other than the SD or USB slot is specified
+     */
+    public synchronized void attachMetadataArchive(File archive, CdjStatus.TrackSourceSlot slot) throws IOException {
+
+        // Determine which slot we are adjusting the archive for.
+        if (slot != CdjStatus.TrackSourceSlot.SD_SLOT && slot != CdjStatus.TrackSourceSlot.USB_SLOT) {
+            throw new IllegalArgumentException("Unsupported slot, use SD_SLOT for USB 1 or USB_SLOT for USB 2: " + slot);
+        }
+
+        final AtomicReference<Database> databaseReference = (slot == CdjStatus.TrackSourceSlot.SD_SLOT) ? usb1database : usb2database;
+        final AtomicReference<FileSystem> filesystemReference = (slot == CdjStatus.TrackSourceSlot.SD_SLOT) ? usb1filesystem : usb2filesystem;
+
+        // First close and remove any archive we had previously attached for this slot.
+        final Database formerDatabase = databaseReference.getAndSet(null);
+        if (formerDatabase != null) {
+            try {
+                formerDatabase.close();
+                //noinspection ResultOfMethodCallIgnored
+                formerDatabase.sourceFile.delete();
+            } catch (IOException e) {
+                logger.error("Problem closing database for " + slot, e);
+            }
+        }
+        final FileSystem formerArchive = filesystemReference.getAndSet(null);
+        if (formerArchive != null) {
+            try {
+                logger.info("Detached metadata archive {} from slot {}", formerArchive, slot);
+                formerArchive.close();
+            } catch (IOException e) {
+                logger.error("Problem closing archive filesystem for USB 1", e);
+            }
+
+            // Clean up any extracted files associated with this archive.
+            final String prefix = slotPrefix(slot);
+            File[] files = extractDirectory.listFiles();
+            if (files != null) {
+                for (File file : files) {
+                    if (file.getName().startsWith(prefix)) {
+                        //noinspection ResultOfMethodCallIgnored
+                        file.delete();
+                    }
+                }
+            }
+        }
+
+        if (archive == null) return;  // Detaching is all we were asked to do, so we are done.
+
+        // Open the new archive filesystem.
+        FileSystem filesystem = FileSystems.newFileSystem(archive.toPath(), Thread.currentThread().getContextClassLoader());
+        try {
+            final File databaseFile = new File(extractDirectory, slotPrefix(slot) + "export.pdb");
+            Files.copy(filesystem.getPath("/export.pdb"), databaseFile.toPath());
+            final Database database = new Database(databaseFile);
+            // If we got here, this looks like a valid metadata archive because we found a valid database export inside it.
+            databaseReference.set(database);
+            filesystemReference.set(filesystem);
+            logger.info("Attached metadata archive {} for slot {}.", filesystem, slot);
+        } catch (Exception e) {
+            filesystem.close();
+            throw new IOException("Problem reading export.pdb from metadata archive " + archive, e);
+        }
+    }
+
+    /**
+     * Find the database we have been provided and parsed that can provide information about the supplied slot
+     * reference, if any (we follow the XDJ-XZ convention of using {@link CdjStatus.TrackSourceSlot#SD_SLOT}
+     * to represent USB 1, and {@link CdjStatus.TrackSourceSlot#USB_SLOT} to represent USB 2).
+     *
+     * @param reference identifies the location for which data is desired
+     *
+     * @return the appropriate rekordbox extract to start from in finding that data, if we have one
+     *
+     * @throws IllegalArgumentException if a slot reference other than the SD or USB slot is specified
+     */
+    @SuppressWarnings("WeakerAccess")
+    public Database findDatabase(DataReference reference) {
+        if (reference.player >= 9 && reference.player <= 12) {  // This is an Opus Quad deck.
+            return findDatabase(reference.getSlotReference());
+        }
+        return null;
+    }
+
+    /**
+     * Find the database we have been provided and parsed that can provide information about the supplied slot
+     * reference, if any (we follow the XDJ-XZ convention of using {@link CdjStatus.TrackSourceSlot#SD_SLOT}
+     * to represent USB 1, and {@link CdjStatus.TrackSourceSlot#USB_SLOT} to represent USB 2).
+     *
+     * @param slot identifies the slot for which data is desired
+     *
+     * @return the appropriate rekordbox extract to start from in finding that data, if we have one
+     *
+     * @throws IllegalArgumentException if a slot reference other than the SD or USB slot is specified
+     */
+    public Database findDatabase(SlotReference slot) {
+        switch (slot.slot) {
+            case SD_SLOT:
+                return usb1database.get();
+            case USB_SLOT:
+                return usb2database.get();
+            default:
+                throw new IllegalArgumentException("Unsupported slot, use SD_SLOT for USB 1 or USB_SLOT for USB 2: " + slot.slot);
+        }
+    }
+
+    /**
+     * Find the ZIP filesystem we have been provided that can provide metadata for the supplied slot
+     * reference, if any (we follow the XDJ-XZ convention of using {@link CdjStatus.TrackSourceSlot#SD_SLOT}
+     * to represent USB 1, and {@link CdjStatus.TrackSourceSlot#USB_SLOT} to represent USB 2).
+     *
+     * @param reference identifies the location for which metadata is desired
+     *
+     * @return the appropriate archive ZIP filesystem holding that metadata, if we have one
+     *
+     * @throws IllegalArgumentException if a slot reference other than the SD or USB slot is specified
+     */
+    public FileSystem findFilesystem(DataReference reference) {
+        return findFilesystem(reference.getSlotReference());
+    }
+
+    /**
+     * Find the ZIP filesystem we have been provided that can provide metadata for the supplied slot
+     * reference, if any (we follow the XDJ-XZ convention of using {@link CdjStatus.TrackSourceSlot#SD_SLOT}
+     * to represent USB 1, and {@link CdjStatus.TrackSourceSlot#USB_SLOT} to represent USB 2).
+     *
+     * @param slot identifies the slot for which metadata is desired
+     *
+     * @return the appropriate archive ZIP filesystem holding that metadata, if we have one
+     *
+     * @throws IllegalArgumentException if a slot reference other than the SD or USB slot is specified
+     */
+    public FileSystem findFilesystem(SlotReference slot) {
+        switch (slot.slot) {
+            case SD_SLOT:
+                return usb1filesystem.get();
+            case USB_SLOT:
+                return usb2filesystem.get();
+            default:
+                throw new IllegalArgumentException("Unsupported slot, use SD_SLOT for USB 1 or USB_SLOT for USB 2: " + slot.slot);
+        }
+    }
+
+    /**
+     * Format the filename prefix that will be used to store files downloaded from a particular USB slot.
+     * This allows them all to be cleaned up when that slot media is detached.
+     *
+     * @param slot the slot from which files are being downloaded
+     *
+     * @return the prefix with which the names of all files downloaded from that slot will start
+     */
+    private String slotPrefix(CdjStatus.TrackSourceSlot slot) {
+        return "slot-" + slot.protocolValue + "-";
+    }
+
+    /**
+     * Find the analysis file for the specified track.
+     * Be sure to call {@code _io().close()} when you are done using the returned struct.
+     *
+     * @param track the track whose analysis file is desired
+     * @param database the parsed database export from which the analysis path can be determined
+     * @param filesystem the open ZIP filesystem in which metadata can be found
+     *
+     * @return the parsed file containing the track analysis
+     */
+    private RekordboxAnlz findTrackAnalysis(DataReference track, Database database, FileSystem filesystem) {
+        return findTrackAnalysis(track, database, filesystem, ".DAT");
+    }
+
+    /**
+     * Find the extended analysis file for the specified track.
+     * Be sure to call {@code _io().close()} when you are done using the returned struct.
+     *
+     * @param track the track whose extended analysis file is desired
+     * @param database the parsed database export from which the analysis path can be determined
+     * @param filesystem the open ZIP filesystem in which metadata can be found
+     *
+     * @return the parsed file containing the track analysis
+     */
+    private RekordboxAnlz findExtendedAnalysis(DataReference track, Database database, FileSystem filesystem) {
+        return findTrackAnalysis(track, database, filesystem, ".EXT");
+    }
+
+    /**
+     * Find an analysis file for the specified track, with the specified file extension.
+     * Be sure to call {@code _io().close()} when you are done using the returned struct.
+     *
+     * @param track the track whose extended analysis file is desired
+     * @param database the parsed database export from which the analysis path can be determined
+     * @param filesystem the open ZIP filesystem in which metadata can be found
+     * @param extension the file extension (such as ".DAT" or ".EXT") which identifies the type file to be retrieved
+     *
+     * @return the parsed file containing the track analysis
+     */
+    private RekordboxAnlz findTrackAnalysis(DataReference track, Database database, FileSystem filesystem, String extension) {
+        File file = null;
+        try {
+            RekordboxPdb.TrackRow trackRow = database.trackIndex.get((long) track.rekordboxId);
+            if (trackRow != null) {
+                file = new File(extractDirectory, slotPrefix(track.getSlotReference().slot) +
+                        "track-" + track.rekordboxId + "-anlz" + extension.toLowerCase());
+                final String filePath = file.getCanonicalPath();
+                final String analyzePath = Database.getText(trackRow.analyzePath());
+                final String requestedPath = analyzePath.replaceAll("\\.DAT$", extension.toUpperCase());
+                try {
+                    synchronized (Util.allocateNamedLock(filePath)) {
+                        if (file.canRead()) {  // We have already downloaded it.
+                            return new RekordboxAnlz(new RandomAccessFileKaitaiStream(filePath));
+                        }
+                        // Extract it.
+                        Files.copy(filesystem.getPath(requestedPath), file.toPath());  // TODO function that can handle .PIONEER HFS+ variant?
+                        file.deleteOnExit();
+                        return new RekordboxAnlz(new RandomAccessFileKaitaiStream(filePath));
+                    }
+                } catch (Exception e) {  // We can give a more specific error including the file path.
+                    logger.error("Problem parsing requested analysis file " + requestedPath + " for track " + track +
+                            " from database " + database, e);
+                    //noinspection ResultOfMethodCallIgnored
+                    file.delete();
+                } finally {
+                    Util.freeNamedLock(filePath);
+                }
+            } else {
+                logger.warn("Unable to find track " + track + " in database " + database);
+            }
+        } catch (Exception e) {
+            logger.error("Problem extracting analysis file with extension " + extension.toUpperCase() +
+                    " for track " + track + " from database " + database, e);
+            if (file != null) {
+                //noinspection ResultOfMethodCallIgnored
+                file.delete();
+            }
+        }
+        return null;
+    }
+
     /**
      * This is the mechanism by which we offer metadata to the {@link MetadataProvider} while we are running.
      */
-    private final MetadataProvider metadataProvider = new MetadataProvider() {
+    public final MetadataProvider metadataProvider = new MetadataProvider() {
         @Override
         public List<MediaDetails> supportedMedia() {
-            return Collections.emptyList();
+            return Collections.emptyList();  // We can answer queries about any media.
         }
 
         @Override
         public TrackMetadata getTrackMetadata(MediaDetails sourceMedia, DataReference track) {
-            // TODO implement!
+            final Database database = findDatabase(track);
+            if (database != null) {
+                try {
+                    return new TrackMetadata(track, database, getCueList(sourceMedia, track));
+                } catch (Exception e) {
+                    logger.error("Problem fetching metadata for track " + track + " from database " + database, e);
+                }
+            }
             return null;
         }
 
@@ -91,9 +352,27 @@ public BeatGrid getBeatGrid(MediaDetails sourceMedia, DataReference track) {
 
         @Override
         public CueList getCueList(MediaDetails sourceMedia, DataReference track) {
-            // TODO implement!
-            return null;
-        }
+            final Database database = findDatabase(track);
+            final FileSystem fileSystem = findFilesystem(track);
+            if (database != null) {
+                try {
+                    // Try the extended file first, because it can contain both nxs2-style commented cues and basic cues
+                    RekordboxAnlz file = findExtendedAnalysis(track, database, fileSystem);
+                    if (file ==  null) {  // No extended analysis found, fall back to the basic one
+                        file = findTrackAnalysis(track, database, fileSystem);
+                    }
+                    if (file != null) {
+                        try {
+                            return new CueList(file);
+                        } finally {
+                            file._io().close();
+                        }
+                    }
+                } catch (Exception e) {
+                    logger.error("Problem fetching cue list for track " + track + " from database " + database, e);
+                }
+            }
+            return null;        }
 
         @Override
         public WaveformPreview getWaveformPreview(MediaDetails sourceMedia, DataReference track) {
@@ -150,11 +429,17 @@ public static OpusProvider getInstance() {
         return instance;
     }
 
+    /**
+     * The folder into which database exports and track analysis files will be extracted.
+     */
+    @SuppressWarnings("WeakerAccess")
+    public final File extractDirectory;
+
     /**
      * Prevent direct instantiation.
      */
     private OpusProvider() {
-        // Nothing to do.
+        extractDirectory = CrateDigger.createDownloadDirectory();
     }
 
     @Override

src/main/java/org/deepsymmetry/beatlink/data/SlotReference.java

@@ -55,17 +55,8 @@ private SlotReference(int player, CdjStatus.TrackSourceSlot slot) {
      */
     @SuppressWarnings("WeakerAccess")
     public static synchronized SlotReference getSlotReference(int player, CdjStatus.TrackSourceSlot slot) {
-        Map<CdjStatus.TrackSourceSlot, SlotReference> playerMap = instances.get(player);
-        if (playerMap == null) {
-            playerMap = new HashMap<CdjStatus.TrackSourceSlot, SlotReference>();
-            instances.put(player, playerMap);
-        }
-        SlotReference result = playerMap.get(slot);
-        if (result == null) {
-            result = new SlotReference(player, slot);
-            playerMap.put(slot, result);
-        }
-        return result;
+        Map<CdjStatus.TrackSourceSlot, SlotReference> playerMap = instances.computeIfAbsent(player, k -> new HashMap<CdjStatus.TrackSourceSlot, SlotReference>());
+        return playerMap.computeIfAbsent(slot, s -> new SlotReference(player, s));
     }
 
     /**