diff --git a/README.md b/README.md index e91e51d..2c27d99 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,22 @@ IDE of choice. The current source needs to be compiled with JDK 1.6, or later. T Note: If you intend to modify user interface code, then you will need to use JFormdesigner to open the *.jfd files. Consult the JFormDesigner documentation on how to install and use it. +NOTES FOR DEVELOPERS + +The majority of development work will probably take place in a few classes, primarily in the package +at-migration\src\org\archiviststoolkit\plugin\utils\aspace +A description of some of the classes is given below. + +1. dbCopyFrame is the user interface for the migration tool. It mostly calls methods in ASpaceCopyUtil. +2. dbCopyCLI mainly does the same thing as dbCopyFrame but is for command line mode. +3. ASpaceCopyUtil has a method for copying each record type. Each of these is called by dbCopyFrame/dbCopyCLI. It + gets records from AT, calls on ASpaceMapper to map them to the ASpace model, then calls on ASpaceClient to save + them to ASpace. +4. ASpaceMapper takes AT model records and converts them to the appropriate ASpace JSON model. +5. TopContainerMapper serves the same function as ASpaceMapper but is specialized for top containers, which present + some special challenges. +6. ASpaceClient saves records to ASpace and gets records from ASpace. + NOTES ON TESTING When writing tests you should always place them in the test directory. To write a test: @@ -30,7 +46,7 @@ When writing tests you should always place them in the test directory. To write 2. Import junit.framework.JUnit4TestAdapter, org.junit.Assert, and org.junit.Test. 3. Write tests with annotation @Test in this class. 4. Add the following method: - public static junit.framework.Test suite() {return new JUnit4TestAdapter(*Name of your class here*.class);} + public static junit.framework.Test suite() {return new JUnit4TestAdapter(*Name of your class here*.class);} 5. You can now run your test by calling a constructor for your class from the testing main method then running the Testing main method. @@ -81,7 +97,7 @@ HOW TOP CONTAINER UNIQUENESS IS DETERMINED Archivist's Toolkit does not include the concept of distinct top containers but rather the same container may be entered many times for multiple instances. For this reason it was necessary to determine a set of rules for determining what constitutes a unique top container. For top containers created from instances, they are considered -to be the same if either the repository and barcode match or if the repository, indicator, and container type +to be the same if either the repository and barcode match or if the repository, resource, indicator, and container type match. As for accessions, no instances exist in Archivist's Toolkit. In stead, a top container is created for each location the accession is linked to. diff --git a/docs/AT Migration Instructions.docx b/docs/AT Migration Instructions.docx new file mode 100644 index 0000000..eb0a56e Binary files /dev/null and b/docs/AT Migration Instructions.docx differ diff --git a/docs/AT Migration Instructions.pdf b/docs/AT Migration Instructions.pdf new file mode 100644 index 0000000..5cc81d0 Binary files /dev/null and b/docs/AT Migration Instructions.pdf differ diff --git a/docs/ATMappingDocument.xlsx b/docs/ATMappingDocument.xlsx new file mode 100644 index 0000000..268bfb0 Binary files /dev/null and b/docs/ATMappingDocument.xlsx differ diff --git a/src/org/archiviststoolkit/plugin/ScriptATFrame.java b/src/org/archiviststoolkit/plugin/ScriptATFrame.java index ccc5bd7..e4e9048 100644 --- a/src/org/archiviststoolkit/plugin/ScriptATFrame.java +++ b/src/org/archiviststoolkit/plugin/ScriptATFrame.java @@ -76,7 +76,7 @@ private void dbCopyButtonActionPerformed() { private void initComponents() { // JFormDesigner - Component initialization - DO NOT MODIFY //GEN-BEGIN:initComponents - // Generated using JFormDesigner Evaluation license - Sarah Morrissey + // Generated using JFormDesigner non-commercial license dialogPane = new JPanel(); contentPanel = new JPanel(); showScriptConsoleButton = new JButton(); @@ -93,14 +93,6 @@ private void initComponents() { //======== dialogPane ======== { dialogPane.setBorder(new EmptyBorder(12, 12, 12, 12)); - - // JFormDesigner evaluation mark - dialogPane.setBorder(new javax.swing.border.CompoundBorder( - new javax.swing.border.TitledBorder(new javax.swing.border.EmptyBorder(0, 0, 0, 0), - "JFormDesigner Evaluation", javax.swing.border.TitledBorder.CENTER, - javax.swing.border.TitledBorder.BOTTOM, new java.awt.Font("Dialog", java.awt.Font.BOLD, 12), - java.awt.Color.red), dialogPane.getBorder())); dialogPane.addPropertyChangeListener(new java.beans.PropertyChangeListener(){public void propertyChange(java.beans.PropertyChangeEvent e){if("border".equals(e.getPropertyName()))throw new RuntimeException();}}); - dialogPane.setLayout(new BorderLayout()); //======== contentPanel ======== @@ -178,7 +170,7 @@ public void actionPerformed(ActionEvent e) { } // JFormDesigner - Variables declaration - DO NOT MODIFY //GEN-BEGIN:variables - // Generated using JFormDesigner Evaluation license - Sarah Morrissey + // Generated using JFormDesigner non-commercial license private JPanel dialogPane; private JPanel contentPanel; private JButton showScriptConsoleButton; diff --git a/src/org/archiviststoolkit/plugin/beanshell/ScriptManagerDialog.java b/src/org/archiviststoolkit/plugin/beanshell/ScriptManagerDialog.java index b005829..b7269cf 100644 --- a/src/org/archiviststoolkit/plugin/beanshell/ScriptManagerDialog.java +++ b/src/org/archiviststoolkit/plugin/beanshell/ScriptManagerDialog.java @@ -150,6 +150,7 @@ private void initComponents() { //---- scriptList ---- scriptList.setSelectionMode(ListSelectionModel.SINGLE_SELECTION); scriptList.addListSelectionListener(new ListSelectionListener() { + @Override public void valueChanged(ListSelectionEvent e) { scriptListValueChanged(e); } @@ -185,6 +186,7 @@ public void valueChanged(ListSelectionEvent e) { //---- storeButton ---- storeButton.setText("Store/Update"); storeButton.addActionListener(new ActionListener() { + @Override public void actionPerformed(ActionEvent e) { storeButtonActionPerformed(); } @@ -196,6 +198,7 @@ public void actionPerformed(ActionEvent e) { //---- viewButton ---- viewButton.setText("View"); viewButton.addActionListener(new ActionListener() { + @Override public void actionPerformed(ActionEvent e) { viewButtonActionPerformed(); } @@ -207,6 +210,7 @@ public void actionPerformed(ActionEvent e) { //---- deleteButton ---- deleteButton.setText("Delete"); deleteButton.addActionListener(new ActionListener() { + @Override public void actionPerformed(ActionEvent e) { deleteButtonActionPerformed(); } @@ -218,6 +222,7 @@ public void actionPerformed(ActionEvent e) { //---- okButton ---- okButton.setText("OK"); okButton.addActionListener(new ActionListener() { + @Override public void actionPerformed(ActionEvent e) { okButtonActionPerformed(); } diff --git a/src/org/archiviststoolkit/plugin/dbCopyCLI.java b/src/org/archiviststoolkit/plugin/dbCopyCLI.java index 931dac4..4b9d24e 100644 --- a/src/org/archiviststoolkit/plugin/dbCopyCLI.java +++ b/src/org/archiviststoolkit/plugin/dbCopyCLI.java @@ -15,9 +15,9 @@ * A class which allows the migration tool to be run from the command line * * Created by IntelliJ IDEA. - * User: nathan - * Date: 4/21/14 - * Time: 2:58 PM + * @author nathan + * Updated by Sarah Morrissey 12/17 + * @version 2.2 */ public class dbCopyCLI { // these fields are read in from the properties file @@ -237,16 +237,15 @@ private void startASpaceCopyProcess() { } else { System.out.println("Administrator authenticated ...\n"); } -// -// // first load the notes etc types and resource from the destination database -// ascopy.loadRepositories(); + + // first load the notes etc types and resource from the destination database if (continueFromURIMaps && ascopy.uriMapFileExist()) { ascopy.loadURIMaps(); } else { ascopy.loadRepositories(); } -// if(!copyOnlyResources) { + ascopy.copyLookupList(); ascopy.copyRepositoryRecords(); ascopy.mapRepositoryGroups(); @@ -259,11 +258,6 @@ private void startASpaceCopyProcess() { ascopy.copyAccessionRecords(); ascopy.copyDigitalObjectRecords(); - // save the record maps for possible future use -// ascopy.saveURIMaps(); -// } -// } - // set the number of resources to copy int numberOfResourcesToCopy = 1000000; @@ -274,8 +268,6 @@ private void startASpaceCopyProcess() { ascopy.copyResourceRecords(numberOfResourcesToCopy, clientThreads); -// ascopy.addContainerData(); - ascopy.addAssessments(); // DEBUG code which checks to see that all ISO dates are valid diff --git a/src/org/archiviststoolkit/plugin/dbCopyFrame.java b/src/org/archiviststoolkit/plugin/dbCopyFrame.java index 0f551d5..6f307ff 100644 --- a/src/org/archiviststoolkit/plugin/dbCopyFrame.java +++ b/src/org/archiviststoolkit/plugin/dbCopyFrame.java @@ -9,7 +9,6 @@ import com.jgoodies.forms.layout.*; import org.archiviststoolkit.ApplicationFrame; import org.archiviststoolkit.importer.ImportExportLogDialog; -import org.archiviststoolkit.model.LookupList; import org.archiviststoolkit.plugin.beanshell.ScriptViewerDialog; import org.archiviststoolkit.plugin.dbdialog.RemoteDBConnectDialogLight; import org.archiviststoolkit.plugin.utils.CodeViewerDialog; @@ -24,8 +23,7 @@ import java.awt.*; import java.awt.event.ActionEvent; import java.awt.event.ActionListener; -import java.io.PrintWriter; -import java.io.StringWriter; +import java.io.*; import java.util.ArrayList; import java.util.HashMap; @@ -33,9 +31,12 @@ * Simple class to test the database transfer code without starting of the AT client application * * @author Nathan Stevens + * updated by Sarah Morrissey 12/2017 + * @version 2.2 */ public class dbCopyFrame extends JFrame { public static final String VERSION = "Archives Space Data Migrator v2.x (10/2017)"; + public static FileOutputStream fw; // The application when running within the AT private ApplicationFrame mainFrame = null; @@ -68,6 +69,7 @@ public class dbCopyFrame extends JFrame { // used to find and attempt to resolve repository mismatch private boolean checkRepositoryMismatch = false; + private boolean continueMigration = false; private HashMap repositoryMismatchMap = null; // used to specify that the GUI is in basic mode @@ -99,35 +101,31 @@ public dbCopyFrame(ApplicationFrame mainFrame, boolean basic) { } } + public static void saveConsoleText(JTextArea console) { + if (fw != null && console != null) { + try { + fw.write((console.getText() + "\n\n").getBytes()); + } catch (IOException e) { + } + } + } + /** * Method to hide advance UI features to make it easier for users to run the tool */ private void hideAdvanceFeatures() { - //apiLabel.setVisible(false); sourceLabel.setVisible(false); sourceTextField.setVisible(false); - //threadLabel.setVisible(false); threadsTextField.setEnabled(false); - //repositoryCheckButton.setVisible(false); - //copyRecordCheckBox.setVisible(false); - //viewRepositoryCheckReportButton.setVisible(false); adminLabel.setVisible(false); adminTextField.setVisible(false); tracerPanel.setVisible(false); - //useSaveURIMapsCheckBox.setVisible(false); simulateCheckBox.setVisible(false); useScriptCheckBox.setVisible(false); editScriptButton.setVisible(false); - //ignoreUnlinkedRecordsLabel.setVisible(false); - //ignoreUnlinkedNamesCheckBox.setVisible(false); - //ignoreUnlinkedSubjectsCheckBox.setVisible(false); - //publishPanel.setVisible(false); batchImportCheckBox.setVisible(false); -// deleteResourcesCheckBox.setVisible(false); numResourceToCopyLabel.setVisible(false); numResourceToCopyTextField.setVisible(false); -// deleteResourcesCheckBox.setVisible(false); - //resourcesToCopyTextField.setVisible(false); recordURIComboBox.setVisible(false); paramsLabel.setVisible(false); paramsTextField.setVisible(false); @@ -142,6 +140,9 @@ private void hideAdvanceFeatures() { * Close this window, and only exit if we are running in stand alone mode */ private void okButtonActionPerformed() { + + saveConsoleText(consoleTextArea); + setVisible(false); if (mainFrame == null && !isBasicUI) { @@ -156,6 +157,7 @@ private void CopyToASpaceButtonActionPerformed() { // first check that the user is running update 15, or greater if (mainFrame != null && !mainFrame.getAtVersionNumber().contains("15") && !mainFrame.getAtVersionNumber().contains("16")) { + saveConsoleText(consoleTextArea); consoleTextArea.setText("You need AT version 2.0 Update 15, or greater for data migration ..."); return; } @@ -190,7 +192,13 @@ private void CopyToASpaceButtonActionPerformed() { } } - // Method to open the dialog that gets a session + /** + * Method to open the dialog that gets a session + * @param title + * @param urlIndex + * @param reuse + * @return + */ public Session displayRemoteConnectionDialog(String title, int urlIndex, boolean reuse) { if (reuse && storedRCDS.get(title) != null) { RemoteDBConnectDialogLight rcd = storedRCDS.get(title); @@ -251,6 +259,7 @@ public void run() { stopButton.setEnabled(true); // clear text area and show progress bar + saveConsoleText(consoleTextArea); consoleTextArea.setText(""); copyProgressBar.setStringPainted(true); copyProgressBar.setString("Copying Records ..."); @@ -345,7 +354,7 @@ public void run() { - if (useSaveURIMapsCheckBox.isSelected() && ascopy.uriMapFileExist()) { + if (continueMigration && ascopy.uriMapFileExist()) { ascopy.loadURIMaps(); } else { // first load the notes etc types and resource from the destination database if not using saved ones @@ -373,9 +382,7 @@ public void run() { try { boolean useBatchImport = batchImportCheckBox.isSelected(); -// boolean deleteSavedResources = deleteResourcesCheckBox.isSelected(); ascopy.setUseBatchImport(useBatchImport); -// ascopy.setDeleteSavedResources(deleteSavedResources); // get the number of threads to run the copy process in threads = Integer.parseInt(threadsTextField.getText()); @@ -405,11 +412,13 @@ public void run() { errorCountLabel.setText(errorCount); migrationErrors = ascopy.getSaveErrorMessages() + "\n\nTotal errors/warnings: " + errorCount; } catch (IntentionalExitException e) { + saveConsoleText(consoleTextArea); consoleTextArea.setText(e.getMessage()); consoleTextArea.append("\nWill attempt to save URI maps ..."); if (ascopy != null) ascopy.saveURIMaps(); else consoleTextArea.append("\nCould not save URI maps ...\nMigration will need to be restarted ..."); } catch (Exception e) { + saveConsoleText(consoleTextArea); consoleTextArea.setText("Unrecoverable exception, migration stopped ...\n\n"); if(ascopy != null) { @@ -420,7 +429,7 @@ public void run() { } consoleTextArea.append(getStackTrace(e)); - //e.printStackTrace(); + } finally { sourceRCD.closeSession(); } @@ -460,10 +469,10 @@ public void run() { // first disable/enable the relevant buttons copyToASpaceButton.setEnabled(false); repositoryCheckButton.setEnabled(false); - //errorLogButton.setEnabled(false); stopButton.setEnabled(true); // clear text area and show progress bar + saveConsoleText(consoleTextArea); consoleTextArea.setText(""); copyProgressBar.setStringPainted(true); copyProgressBar.setString("Checking Repository Mismatches ..."); @@ -506,7 +515,7 @@ public void run() { if(!copyStopped) { ascopyREC.setResourcesToCopyList(resourcesIDsList); - ascopyREC.copyResourceRecords(resourcesToCopy, threads); + ascopyREC.checkRepositoryMismatches(resourcesToCopy, threads); } repositoryMismatchMap = ascopyREC.getRepositoryMismatchMap(); @@ -518,6 +527,7 @@ public void run() { errorCountLabel.setText(errorCount); repositoryMismatchErrors = ascopyREC.getCurrentRecordCheckMessage() + "\n\nTotal errors: " + errorCount; } catch (Exception e) { + saveConsoleText(consoleTextArea); consoleTextArea.setText("Unrecoverable exception, recording checking stopped ...\n\n"); // This is null for some reason so let commit it out. @@ -552,7 +562,6 @@ private void reEnableCopyButtons() { // re-enable the buttons the relevant buttons copyToASpaceButton.setEnabled(true); repositoryCheckButton.setEnabled(true); - //errorLogButton.setEnabled(true); copyProgressBar.setValue(0); if (copyStopped) { @@ -711,9 +720,11 @@ private void testRecordButtonActionPerformed() { codeViewerDialog.pack(); codeViewerDialog.setVisible(true); } else { - consoleTextArea.setText("You need AT version 2.0 Update 15, and above for this to work"); + saveConsoleText(consoleTextArea); + consoleTextArea.setText("You need AT version 2.0, and above for this to work"); } } catch (Exception e) { + saveConsoleText(consoleTextArea); consoleTextArea.setText(getStackTrace(e)); } } @@ -748,9 +759,14 @@ private String getStackTrace(Throwable throwable) { return sw.toString(); } + private void continueButtonActionPerformed() { + continueMigration = true; + CopyToASpaceButtonActionPerformed(); + } + private void initComponents() { // JFormDesigner - Component initialization - DO NOT MODIFY //GEN-BEGIN:initComponents - // Generated using JFormDesigner Evaluation license - Sarah Morrissey + // Generated using JFormDesigner non-commercial license dialogPane = new JPanel(); contentPanel = new JPanel(); apiLabel = new JLabel(); @@ -764,14 +780,14 @@ private void initComponents() { repositoryCheckButton = new JButton(); copyRecordCheckBox = new JCheckBox(); viewRepositoryCheckReportButton = new JButton(); - tracerPanel = new JPanel(); - useTracerCheckBox = new JCheckBox(); - tracerComboBox = new JComboBox(); + continueButton = new JButton(); adminLabel = new JLabel(); adminTextField = new JTextField(); adminPasswordLabel = new JLabel(); adminPasswordTextField = new JTextField(); - useSaveURIMapsCheckBox = new JCheckBox(); + tracerPanel = new JPanel(); + useTracerCheckBox = new JCheckBox(); + tracerComboBox = new JComboBox(); resetPassswordLabel = new JLabel(); resetPasswordTextField = new JTextField(); typeOfExtentDataLabel = new JLabel(); @@ -794,7 +810,6 @@ private void initComponents() { batchImportCheckBox = new JCheckBox(); numResourceToCopyLabel = new JLabel(); numResourceToCopyTextField = new JTextField(); -// deleteResourcesCheckBox = new JCheckBox(); resourcesToCopyTextField = new JTextField(); outputConsoleLabel = new JLabel(); copyProgressBar = new JProgressBar(); @@ -822,14 +837,6 @@ private void initComponents() { //======== dialogPane ======== { dialogPane.setBorder(Borders.DIALOG_BORDER); - - // JFormDesigner evaluation mark - dialogPane.setBorder(new javax.swing.border.CompoundBorder( - new javax.swing.border.TitledBorder(new javax.swing.border.EmptyBorder(0, 0, 0, 0), - "JFormDesigner Evaluation", javax.swing.border.TitledBorder.CENTER, - javax.swing.border.TitledBorder.BOTTOM, new java.awt.Font("Dialog", java.awt.Font.BOLD, 12), - java.awt.Color.red), dialogPane.getBorder())); dialogPane.addPropertyChangeListener(new java.beans.PropertyChangeListener(){public void propertyChange(java.beans.PropertyChangeEvent e){if("border".equals(e.getPropertyName()))throw new RuntimeException();}}); - dialogPane.setLayout(new BorderLayout()); //======== contentPanel ======== @@ -948,24 +955,15 @@ public void actionPerformed(ActionEvent e) { }); contentPanel.add(viewRepositoryCheckReportButton, cc.xywh(11, 5, 3, 1)); - //======== tracerPanel ======== - { - tracerPanel.setLayout(new FlowLayout(FlowLayout.LEFT)); - - //---- useTracerCheckBox ---- - useTracerCheckBox.setText("Use Tracer Database"); - tracerPanel.add(useTracerCheckBox); - - //---- tracerComboBox ---- - tracerComboBox.setModel(new DefaultComboBoxModel(new String[] { - "1", - "2", - "3", - "SB" - })); - tracerPanel.add(tracerComboBox); - } - contentPanel.add(tracerPanel, cc.xy(1, 7)); + //---- continueButton ---- + continueButton.setText("Continue Previous Migration"); + continueButton.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + continueButtonActionPerformed(); + } + }); + contentPanel.add(continueButton, cc.xy(1, 7)); //---- adminLabel ---- adminLabel.setText("Admin"); @@ -983,9 +981,24 @@ public void actionPerformed(ActionEvent e) { adminPasswordTextField.setText("admin"); contentPanel.add(adminPasswordTextField, cc.xy(13, 7)); - //---- useSaveURIMapsCheckBox ---- - useSaveURIMapsCheckBox.setText("Continue Previous Migration"); - contentPanel.add(useSaveURIMapsCheckBox, cc.xy(1, 9)); + //======== tracerPanel ======== + { + tracerPanel.setLayout(new FlowLayout(FlowLayout.LEFT)); + + //---- useTracerCheckBox ---- + useTracerCheckBox.setText("Use Tracer Database"); + tracerPanel.add(useTracerCheckBox); + + //---- tracerComboBox ---- + tracerComboBox.setModel(new DefaultComboBoxModel(new String[] { + "1", + "2", + "3", + "SB" + })); + tracerPanel.add(tracerComboBox); + } + contentPanel.add(tracerPanel, cc.xy(1, 9)); //---- resetPassswordLabel ---- resetPassswordLabel.setText("Reset Password"); @@ -1104,10 +1117,6 @@ public void actionPerformed(ActionEvent e) { numResourceToCopyTextField.setText("100000"); contentPanel.add(numResourceToCopyTextField, cc.xywh(9, 19, 5, 1)); -// //---- deleteResourcesCheckBox ---- -// deleteResourcesCheckBox.setText("Delete Previously Saved Resources"); -// contentPanel.add(deleteResourcesCheckBox, cc.xy(1, 21)); - //---- resourcesToCopyTextField ---- resourcesToCopyTextField.setText("-refid_unique, -term_default"); contentPanel.add(resourcesToCopyTextField, cc.xywh(3, 21, 11, 1)); @@ -1260,7 +1269,7 @@ public void actionPerformed(ActionEvent e) { // JFormDesigner - Variables declaration - DO NOT MODIFY //GEN-BEGIN:variables - // Generated using JFormDesigner Evaluation license - Sarah Morrissey + // Generated using JFormDesigner non-commercial license private JPanel dialogPane; private JPanel contentPanel; private JLabel apiLabel; @@ -1274,14 +1283,14 @@ public void actionPerformed(ActionEvent e) { private JButton repositoryCheckButton; private JCheckBox copyRecordCheckBox; private JButton viewRepositoryCheckReportButton; - private JPanel tracerPanel; - private JCheckBox useTracerCheckBox; - private JComboBox tracerComboBox; + private JButton continueButton; private JLabel adminLabel; private JTextField adminTextField; private JLabel adminPasswordLabel; private JTextField adminPasswordTextField; - private JCheckBox useSaveURIMapsCheckBox; + private JPanel tracerPanel; + private JCheckBox useTracerCheckBox; + private JComboBox tracerComboBox; private JLabel resetPassswordLabel; private JTextField resetPasswordTextField; private JLabel typeOfExtentDataLabel; @@ -1304,7 +1313,6 @@ public void actionPerformed(ActionEvent e) { private JCheckBox batchImportCheckBox; private JLabel numResourceToCopyLabel; private JTextField numResourceToCopyTextField; -// private JCheckBox deleteResourcesCheckBox; private JTextField resourcesToCopyTextField; private JLabel outputConsoleLabel; private JProgressBar copyProgressBar; diff --git a/src/org/archiviststoolkit/plugin/dbCopyFrame.jfd b/src/org/archiviststoolkit/plugin/dbCopyFrame.jfd index f400621..c95aa25 100644 --- a/src/org/archiviststoolkit/plugin/dbCopyFrame.jfd +++ b/src/org/archiviststoolkit/plugin/dbCopyFrame.jfd @@ -332,61 +332,30 @@ - - javax.swing.JPanel - - java.awt.FlowLayout - - alignment - 0 - - - - tracerPanel + + javax.swing.JButton + + text + Continue Previous Migration - - - javax.swing.JCheckBox - - text - Use Tracer Database - - - useTracerCheckBox - - + + continueButton - - - javax.swing.JComboBox - - model - - - 1 - - - 1 - - - 2 - - - 3 - - - SB - - - - - tracerComboBox - + + + java.awt.event.ActionListener + actionPerformed + continueButtonActionPerformed + false com.jgoodies.forms.layout.CellConstraints + + gridX + 1 + gridY 7 @@ -486,14 +455,57 @@ - - javax.swing.JCheckBox - - text - Continue From Resource Records - + + javax.swing.JPanel + + java.awt.FlowLayout + + alignment + 0 + + - useSaveURIMapsCheckBox + tracerPanel + + + + javax.swing.JCheckBox + + text + Use Tracer Database + + + useTracerCheckBox + + + + + + javax.swing.JComboBox + + model + + + 1 + + + 1 + + + 2 + + + 3 + + + SB + + + + + tracerComboBox + + @@ -1077,29 +1089,6 @@ - - - javax.swing.JCheckBox - - text - Delete Previously Saved Resources - - - deleteResourcesCheckBox - - - - com.jgoodies.forms.layout.CellConstraints - - gridX - 1 - - - gridY - 21 - - - javax.swing.JTextField diff --git a/src/org/archiviststoolkit/plugin/dbdialog/RemoteDBConnectDialogLight.java b/src/org/archiviststoolkit/plugin/dbdialog/RemoteDBConnectDialogLight.java index 9313ec6..7da71e0 100644 --- a/src/org/archiviststoolkit/plugin/dbdialog/RemoteDBConnectDialogLight.java +++ b/src/org/archiviststoolkit/plugin/dbdialog/RemoteDBConnectDialogLight.java @@ -739,10 +739,9 @@ public ArrayList getRecords(Class clazz, Session session) { * Method to add more extent types to the AT lookup list in case the user made additions * to the backend * - * @param lookupList * @return */ - public void addExtentTypes(LookupList lookupList) { + public void addExtentTypes(ArrayList additional) { Transaction tx = null; ArrayList recordList = null; @@ -760,7 +759,7 @@ public void addExtentTypes(LookupList lookupList) { String extentType = (String)record; if(!extentType.isEmpty()) { - lookupList.addListItem(extentType); + additional.add(extentType); } } } @@ -769,9 +768,8 @@ public void addExtentTypes(LookupList lookupList) { * Method to add salutations to a dummy lookup list in order to add them to the ASpace * enum for this * - * @param lookupList */ - public void addSalutations(LookupList lookupList) { + public void addSalutations(ArrayList additional) { Transaction tx = null; ArrayList recordList = null; @@ -789,7 +787,7 @@ public void addSalutations(LookupList lookupList) { String salutation = (String)record; if(!salutation.isEmpty()) { - lookupList.addListItem(salutation); + additional.add(salutation); } } } diff --git a/src/org/archiviststoolkit/plugin/utils/aspace/ASpaceClient.java b/src/org/archiviststoolkit/plugin/utils/aspace/ASpaceClient.java index ba25199..070dabc 100644 --- a/src/org/archiviststoolkit/plugin/utils/aspace/ASpaceClient.java +++ b/src/org/archiviststoolkit/plugin/utils/aspace/ASpaceClient.java @@ -18,10 +18,10 @@ /** * Created by IntelliJ IDEA. - * User: nathan - * Date: 9/6/12 - * Time: 3:59 PM - *

+ * @author nathan + * updated by Sarah Morrissey 12/2017 + * @version 2.2 + * * This class handles all posting and reading from the ASpace project */ public class ASpaceClient { @@ -211,11 +211,8 @@ private String executePost(PostMethod post, String idName, String atId, String j int statusCode; try { statusCode = httpclient.executeMethod(post); - if (statusCode == HttpStatus.SC_INTERNAL_SERVER_ERROR) { - appendToErrorBuffer("Problem connecting to server"); - throw new IntentionalExitException("Could not connect to server ...\nFix connection then resume ...", atId); - } } catch (ConnectException e) { + // if the connection is lost we want to pause the migration to fix it boolean ready = false; String message = "Connection has been lost. Check your \n" + "connection to Archives Space and your \n" + @@ -223,6 +220,8 @@ private String executePost(PostMethod post, String idName, String atId, String j "contiue migration. Otherwise press \n" + "'no' to save your URI maps and \n" + "continue this migration later."; + + // the user chooses to fix the connection and continue immediately or save the URI maps and continue later while (!ready) { int result = JOptionPane.showConfirmDialog(null, message, "Lost connection", JOptionPane.YES_NO_OPTION, JOptionPane.ERROR_MESSAGE); @@ -262,7 +261,10 @@ private String executePost(PostMethod post, String idName, String atId, String j JSONArray responseJA = new JSONArray(responseBody); response = responseJA.getJSONObject(responseJA.length() - 1); + // if there is a server error stop migration and save place if (response.toString().contains("Server error: Failed to connect to the database")) { + appendToErrorBuffer("Problem connecting to server\n"); + appendToErrorBuffer(post.getResponseBodyAsString()); throw new IntentionalExitException("Could not connect to server ...\nFix connection then resume ...", atId); } @@ -279,6 +281,7 @@ private String executePost(PostMethod post, String idName, String atId, String j } if (post.getURI().toString().contains(ASSESSMENT_ATTR_DEFNS_ENDPOINT)) { + // these aren't assigned IDs but we don't want to treat it as an error if (response.getString("status").equalsIgnoreCase("updated")) id = "ok"; } else { id = response.getString(idName); @@ -318,12 +321,15 @@ private String executePost(PostMethod post, String idName, String atId, String j // if it a 500 error the ASpace then we may need to add the JSON text if (statusCode == HttpStatus.SC_INTERNAL_SERVER_ERROR) { if (responseBody.contains("PoolTimeout")) { - responseBody = "Error: Sequel Pool Timeout ..."; + responseBody = "Error: Sequel Pool Timeout ...\nPlease Resume ..."; } else if (responseBody.contains("OutOfMemory")) { - responseBody = "Fatal Error: ArchivesSpace Backend Crashed (OutOfMemoryError)\nPlease Restart ..."; + responseBody = "Fatal Error: ArchivesSpace Backend Crashed (OutOfMemoryError)\nPlease Resume Later ..."; } else if (responseBody.contains("ThreadError")) { - responseBody = "Fatal Error: ArchivesSpace Backend Crashed (OutOfStackSpaceError)\nPlease Restart ..."; + responseBody = "Fatal Error: ArchivesSpace Backend Crashed (OutOfStackSpaceError)\nPlease Resume Later ..."; } + + // if there's a server error, stop migration and save place + throw new IntentionalExitException(statusMessage + "\n" + responseBody, atId); } errorBuffer.append("Endpoint: ").append(post.getURI()).append("\n"). @@ -331,7 +337,7 @@ private String executePost(PostMethod post, String idName, String atId, String j append(statusMessage).append("\n").append(responseBody).append("\n\n"); post.releaseConnection(); - throw new Exception(statusMessage); + throw new Exception(statusMessage + "\n" + responseBody); } } finally { // Release current connection to the server @@ -443,7 +449,7 @@ public HashMap loadRepositories() { repos.put(shortName, uri); } - //we don't need to load the top containers if only assessments still need to be copied + // we don't need to load the top containers if only assessments still need to be copied loadExistingTopContainers(repos); return repos; @@ -454,6 +460,13 @@ public HashMap loadRepositories() { return null; } + /** + * loads any top containers that are in ASpace before the migration starts + * this shouldn't be necessary but is done anyway to avoid fatal issues + * + * @param repos + * @throws Exception + */ private void loadExistingTopContainers(HashMap repos) throws Exception { for (String repoURI: repos.values()) { NameValuePair[] params = new NameValuePair[1]; diff --git a/src/org/archiviststoolkit/plugin/utils/aspace/ASpaceCopyUtil.java b/src/org/archiviststoolkit/plugin/utils/aspace/ASpaceCopyUtil.java index 9e723d3..8d57a0c 100644 --- a/src/org/archiviststoolkit/plugin/utils/aspace/ASpaceCopyUtil.java +++ b/src/org/archiviststoolkit/plugin/utils/aspace/ASpaceCopyUtil.java @@ -18,9 +18,10 @@ /** * Created by IntelliJ IDEA. - * User: nathan - * Date: 9/5/12 - * Time: 1:48 PM + * @author nathan + * updated by Sarah Morrissey 12/2017 + * @version 2.2 + * * Utility class for copying data from the AT to Archive Space */ public class ASpaceCopyUtil { @@ -70,8 +71,6 @@ public class ASpaceCopyUtil { // hashmap that maps subjects from old database with copy in new database private HashMap subjectURIMap = new HashMap(); - private HashMap otherSubjectURIMap = new HashMap(); - // hashmap that maps names from old database with copy in new database private HashMap nameURIMap = new HashMap(); @@ -87,6 +86,9 @@ public class ASpaceCopyUtil { // maps AT assessment identifier with uri of copy in AS database private HashMap assessmentURIMap = new HashMap(); + // maps a username to the uri it was saved at + private HashMap userURIMap = new HashMap(); + // hashmap that maps a repo with its term definitions which are in turn mapped with their IDs private HashMap> assessmentAttributeDefinitions = new HashMap>(); @@ -94,7 +96,15 @@ public class ASpaceCopyUtil { // used to determine whether to attempt to copy assessments when ASpace version can not be determined private Boolean copyAssessments; - // returns the uri of a record in ASpace - works with most record types + // used to safeguard against overwriting URI maps before they have been loaded + private boolean shouldSave = false; + + /** + * returns the uri of a record in ASpace - works with most record types + * + * @param record AT record + * @return uri of equivalent ASpace record + */ public String getURIMapping(DomainObject record) { Long recordIdentifier = record.getIdentifier(); @@ -119,6 +129,43 @@ public String getURIMapping(DomainObject record) { } } + /** + * get the uri of an agent that matches what is in an Assessment field + * + * @param content + * @return + * @throws Exception + */ + public JSONObject getMatchingUserAgent(String content) throws Exception { + + // if its a record added for the assessments it will be an agent uri + String agentURI = userURIMap.get("assessments_" + content); + if (agentURI != null) return ASpaceMapper.getReferenceObject(agentURI); + + // otherwise if there is a matching user record get its agent record + content = content.trim().toLowerCase(); + String userURI = userURIMap.get(content); + if (userURI != null) { + try { + JSONObject userJSON = getRecord(userURI); + return userJSON.getJSONObject("agent_record"); + } catch (Exception e) { + return null; + } + } + return null; + } + + /** + * adds the name of an agent created for an assessment to the map so it can be used again + * + * @param name content of a field in assessments + * @param uri + */ + public void addAssessmentsNameToUserURIMap(String name, String uri) { + userURIMap.put("assessments_" + name, uri); + } + // stop watch object for keeping tract of time private StopWatch stopWatch = null; @@ -167,7 +214,7 @@ public String getURIMapping(DomainObject record) { private int mismatchesFixed = 0; private int mismatchesNotFixed = 0; - /* Variables used to save URI mapping to file to make data transfer more efficient */ + // Variables used to save URI mapping to file to make data transfer more efficient // file where the uri maps is saved private static File uriMapFile = null; @@ -177,7 +224,6 @@ public String getURIMapping(DomainObject record) { private final String LOCATION_KEY = "locationURIMap"; private final String USER_KEY = "userURIMap"; private final String SUBJECT_KEY = "subjectURIMap"; - private final String OTHER_SUBJECT_KEY = "otherSubjectURIMap"; private final String NAME_KEY = "nameURIMap"; private final String ACCESSION_KEY = "accessionURIMap"; private final String DIGITAL_OBJECT_KEY = "digitalObjectURIMap"; @@ -483,11 +529,14 @@ public void copyLookupList() throws Exception { return; } + // uri maps definitely have actual data in them now so we want to save them if migration stops + shouldSave = true; + if (!recordsToCopy.contains("Lookup List")) return; ArrayList records = sourceRCD.getLookupLists(); - /* add a dummy lookup list to hold new event fields */ + // add a dummy lookup list to hold new event fields LookupList eventList = new LookupList(); eventList.setListName("Events"); eventList.addListItem("processing_started"); @@ -503,33 +552,31 @@ public void copyLookupList() throws Exception { // these are used to update the progress bar int total = records.size(); int count = numAttempted; - int success = numSuccessful; -// if (recordsToCopy.peek().equals("Lookup List")) { records = new ArrayList(records.subList(numAttempted, total)); -// } for (LookupList lookupList: records) { String listName = lookupList.getListName(); + ArrayList additional = new ArrayList(); - // we may need to add additional values to some lookup list now + // we may need to add additional values in some cases + // don't add items to the actual lookuplist because this actually modifies data in AT if(listName.equalsIgnoreCase("Extent type")) { - sourceRCD.addExtentTypes(lookupList); - lookupList.addListItem("unknown"); + sourceRCD.addExtentTypes(additional); + additional.add("unknown"); } else if(listName.equals("Salutation")) { - sourceRCD.addSalutations(lookupList); + sourceRCD.addSalutations(additional); } else if(listName.equals("Name source")) { - lookupList.addListItem("ingest"); - lookupList.addListItem("local"); + additional.add("unknown"); } else if(listName.equals("Container types")) { - lookupList.addListItem("unknown_item"); + additional.add("unknown_item"); } else if(listName.equals("Resource type")) { - lookupList.addListItem("collection"); + additional.add("collection"); } else if(listName.equals("Date type")) { - lookupList.addListItem("other"); + additional.add("other"); } - JSONObject updatedEnumJS = mapper.mapLookList(lookupList); + JSONObject updatedEnumJS = mapper.mapLookList(lookupList, additional); if (updatedEnumJS != null) { String endpoint = updatedEnumJS.getString("uri"); @@ -538,7 +585,6 @@ public void copyLookupList() throws Exception { if (!id.equalsIgnoreCase(NO_ID)) { print("Copied Lookup List Values: " + lookupList.getListName() + " :: " + id); - success++; numSuccessful++; } } @@ -590,7 +636,7 @@ public void copyRepositoryRecords() throws Exception { // these are used to update the progress bar int total = records.size(); - int count = numAttempted;//0; + int count = numAttempted; int success = numSuccessful; records = new ArrayList(records.subList(numAttempted, total)); @@ -622,8 +668,7 @@ public void copyRepositoryRecords() throws Exception { repositoryAgentURIMap.put(uri, agentURI); print("Copied Repository: " + repository.getShortName() + " :: " + id); -// print("Mapping repository user group records for " + repository.getShortName() + "..."); -// mapRepositoryGroups(uri); + success++; numSuccessful++; } else { @@ -771,10 +816,6 @@ public void copyUserRecords() throws Exception { ArrayList records = sourceRCD.getUsers(); -// if (recordsToCopy.peek().equals("Users")) { -// records = new ArrayList(records.subList(numAttempted, records.size())); -// } - // these are used to update the progress bar int total = records.size(); int count = numAttempted; @@ -805,6 +846,8 @@ public void copyUserRecords() throws Exception { String id = saveRecord(ASpaceClient.USER_ENDPOINT, jsonText, params, "User->" + user.getUserName()); if (!id.equalsIgnoreCase(NO_ID)) { + String uri = ASpaceClient.USER_ENDPOINT + "/" + id; + addUserToURIMap(user, uri); print("Copied User: " + user.toString() + " :: " + id); success++; numSuccessful++; @@ -828,6 +871,19 @@ public void copyUserRecords() throws Exception { freeMemory(); } + /** + * adds 2 mappings - username to user uri and user full name to user uri + * + * @param user + * @param uri + */ + private void addUserToURIMap(Users user, String uri) { + String username = user.getUserName().trim().toLowerCase(); + String name = user.getFullName().trim().toLowerCase(); + userURIMap.put(username, uri); + userURIMap.put(name, uri); + } + /** * Method to copy a test user to the ASpace backend * @@ -918,10 +974,6 @@ public void copyNameRecords() throws Exception { ArrayList records = sourceRCD.getNames(); -// if (recordsToCopy.peek().equals("Names")) { -// records = new ArrayList(records.subList(numAttempted, records.size())); -// } - // these are used to update the progress bar int total = records.size(); int count = numAttempted; @@ -1003,10 +1055,15 @@ public void copyNameRecords() throws Exception { freeMemory(); } + /** + * copys the assessments if ASpace version is 2.2 or later + * @throws Exception + */ public void addAssessments() throws Exception { if (!recordsToCopy.contains("Assessments")) return; + // first figure out if the version of ASpace allows for assessments if (aspaceVersion.contains("2.2")) { System.out.println("Copying assessments ..."); } else if (aspaceVersion.isEmpty()) { @@ -1026,6 +1083,7 @@ public void addAssessments() throws Exception { return; } + // load the assessment attributes that are already in ASpace loadAssessmentAtributeDefinitions(); ArrayList records = sourceRCD.getAssessments(); @@ -1044,31 +1102,32 @@ public void addAssessments() throws Exception { boolean successful = true; - HashSet linkedRepos = new HashSet(); + // get all the repositories that the assessment's linked records are in + HashSet linkedRepoURIs = new HashSet(); for (AssessmentsDigitalObjects digitalObject : assessment.getDigitalObjects()) { - linkedRepos.add(digitalObject.getDigitalObject().getRepository()); + DigitalObjects record = digitalObject.getDigitalObject(); + linkedRepoURIs.add(getRemappedRepositoryURI("digitalObject", record.getIdentifier(), record.getRepository())); } for (AssessmentsResources resource : assessment.getResources()) { - linkedRepos.add(resource.getResource().getRepository()); + linkedRepoURIs.add(getRepositoryURI(resource.getResource().getRepository())); } for (AssessmentsAccessions accession : assessment.getAccessions()) { - linkedRepos.add(accession.getAccession().getRepository()); + Accessions record = accession.getAccession(); + linkedRepoURIs.add(getRemappedRepositoryURI("accession", record.getIdentifier(), record.getRepository())); } /* since the repositories of all records attached to an assessment must be the same as the assessment's repository it is necessary to add a separate assessment for each linked repository */ - for (Repositories repo: linkedRepos) { - - assessment.setRepository(repo); + for (String repoURI: linkedRepoURIs) { - String repoURI = getRemappedRepositoryURI("assessment", assessment.getIdentifier(), assessment.getRepository()); String uri = repoURI + ASpaceClient.ASSESSMENT_ENDPOINT; - String repoJSON = mapper.addAssessmentsRecords(jsonText, assessment); -// String jsonText = (String) mapper.convert(assessment); + // add linked records and assessment attributes + String repoJSON = mapper.addAssessmentsRepoSpecificInfo(jsonText, assessment, repoURI); + if (jsonText != null) { String id = saveRecord(uri, repoJSON, "Assessment->" + assessment.getAssessmentId()); @@ -1076,8 +1135,6 @@ public void addAssessments() throws Exception { uri = uri + "/" + id; assessmentURIMap.put(assessment.getIdentifier(), uri); print("Copied Assessment: " + assessment + " :: " + id); -// success++; -// numSuccessful++; } else { print("Fail -- Assessment: " + assessment); successful = false; @@ -1152,21 +1209,20 @@ private void loadAssessmentAtributeDefinitions() throws Exception { /** * gets the mapping for a assessment attribute definition if it already exists. Otherwise adds the definition - * @param repo + * @param repoURI * @param label * @param type * @return */ - public Integer getAssessmentAttributeID(Repositories repo, String label, String type) throws Exception { + public Integer getAssessmentAttributeID(String repoURI, String label, String type) throws Exception { Integer id; String key = label + " - " + type; //first see if it is in the global assessment attributes list - id = assessmentAttributeDefinitions.get(aspaceClient.ADMIN_REPOSITORY_ENDPOINT).get(key); + id = assessmentAttributeDefinitions.get(ASpaceClient.ADMIN_REPOSITORY_ENDPOINT).get(key); if (id != null) return id; //next see if it is in the repository specific assessment attributes list - String repoURI = repositoryURIMap.get(repo.getShortName()); id = assessmentAttributeDefinitions.get(repoURI).get(key); if (id != null) return id; @@ -1174,14 +1230,14 @@ public Integer getAssessmentAttributeID(Repositories repo, String label, String JSONObject defn = new JSONObject(); defn.put("label", label); defn.put("type", type); - JSONObject previous = new JSONObject(aspaceClient.get(repoURI + aspaceClient.ASSESSMENT_ATTR_DEFNS_ENDPOINT, + JSONObject previous = new JSONObject(aspaceClient.get(repoURI + ASpaceClient.ASSESSMENT_ATTR_DEFNS_ENDPOINT, null)); JSONArray previousJA = (JSONArray) previous.get("definitions"); id = nextAssesAttrDefnID++; previousJA.put(defn); JSONObject json = new JSONObject(); json.put("definitions", previousJA); - saveRecord(repoURI + aspaceClient.ASSESSMENT_ATTR_DEFNS_ENDPOINT, json.toString(), + saveRecord(repoURI + ASpaceClient.ASSESSMENT_ATTR_DEFNS_ENDPOINT, json.toString(), "Assessment Attributes->" + type); assessmentAttributeDefinitions.get(repoURI).put(key, id); return id; @@ -1200,10 +1256,6 @@ public void copySubjectRecords() throws Exception { ArrayList records = sourceRCD.getSubjects(); -// if (recordsToCopy.peek().equals("Subjects")) { -// records = new ArrayList(records.subList(numAttempted, records.size())); -// } - // these are used to update the progress bar int total = records.size(); int count = numAttempted; @@ -1232,32 +1284,18 @@ public void copySubjectRecords() throws Exception { } String jsonText; - String otherURIMapKey = subject.getSubjectTerm() + " " + enumUtil.getASpaceSubjectSource(subject.getSubjectSource())[0]; String uri; - boolean alreadyExists = otherSubjectURIMap.containsKey(otherURIMapKey); - if (alreadyExists) { - uri = otherSubjectURIMap.get(otherURIMapKey); - try { - JSONObject json = getRecord(uri); - json = mapper.addSubjectTerm(subject, json); - jsonText = json.toString(); - } catch (NullPointerException e) { - jsonText = (String) mapper.convert(subject); - uri = ASpaceClient.SUBJECT_ENDPOINT; - } - } else { jsonText = (String) mapper.convert(subject); uri = ASpaceClient.SUBJECT_ENDPOINT; - } if (jsonText != null) { + String id = saveRecord(uri, jsonText, "Subject->" + subject.getSubjectTerm()); if(!id.equalsIgnoreCase(NO_ID)) { - if (!alreadyExists) uri = ASpaceClient.SUBJECT_ENDPOINT + "/" + id; + uri = ASpaceClient.SUBJECT_ENDPOINT + "/" + id; subjectURIMap.put(subject.getIdentifier(), uri); - if (!alreadyExists) otherSubjectURIMap.put(otherURIMapKey, uri); print("Copied Subject: " + subject + " :: " + id); success++; numSuccessful++; @@ -1265,7 +1303,6 @@ public void copySubjectRecords() throws Exception { print("Fail -- Subject: " + subject); } } else { - print("Fail -- Subject to JSON: " + subject); } @@ -1306,10 +1343,6 @@ public void copyAccessionRecords() throws Exception { ArrayList records = sourceRCD.getAccessions(); -// if (recordsToCopy.peek().equals("Accessions")) { -// records = new ArrayList(records.subList(numAttempted, records.size())); -// } - // these are used to update the progress bar int total = records.size(); int count = numAttempted; @@ -1317,7 +1350,7 @@ public void copyAccessionRecords() throws Exception { records = new ArrayList(records.subList(numAttempted, total)); -// updateProgress("Accessions", total, count); + TopContainerMapper.resetAlreadyAdded(-1); for (Accessions accession : records) { if(stopCopy) return; @@ -1393,7 +1426,6 @@ private void addEvents(Accessions accession, String repoURI, String accessionURI for (JSONObject eventJS: eventList) { String id = saveRecord(uri, eventJS.toString(), "Accession Event->" + accession.getAccessionNumber()); - //System.out.println("Aspace Event ID:" + id); } } @@ -1440,10 +1472,6 @@ public void copyDigitalObjectRecords() throws Exception { ArrayList records = sourceRCD.getDigitalObjects(); -// if (recordsToCopy.peek().equals("Digital Objects")) { -// records = new ArrayList(records.subList(numAttempted, records.size())); -// } - // these are used to update the progress bar int total = records.size(); int count = numAttempted; @@ -1597,6 +1625,71 @@ public void copyDigitalObjectChildren(String endpoint, String digitalObjectURI, } } + public void checkRepositoryMismatches(int max, int threads) throws Exception { + currentRecordType = "Resource Record"; + + if (!recordsToCopy.contains("Resource Records")) return; + + ArrayList records = sourceRCD.getResources(); + + // keep track of the number of resource records copied + copyCount = 0; + + // these are used to update the progress bar + int total = records.size(); + int count = 0; + + print("Checking " + records.size() + " Resource records ..."); + + if(debug && max < total) total = max; + + for (Resources resource : records) { + // we need to update the progress bar here + updateProgress("Resource Records", total, count); + + count++; + + if (stopCopy) { + updateRecordTotals("Resource Records", total, copyCount); + return; + } + + // indicate we are copying the resource record + print("Checking Resource: " + resource.getTitle()); + + // get the parent repository + Repositories repository = resource.getRepository(); + + // these methods find the repository mismatches + addInstances(null, resource, repository, null); + addRelatedAccessions(null, resource, null); + + // add any archival objects here + Set resourceComponents = resource.getResourcesComponents(); + + for (ResourcesComponents component : resourceComponents) { + checkComponentRepoMismatches(component, repository); + } + + incrementCopyCount(); + updateRecordTotals("Resource Records", total, copyCount); + } + } + + /** + * method to make sure none of the resource components have mismatches with their linked digital objects + * @param component + * @param repository + * @throws Exception + */ + private void checkComponentRepoMismatches(ResourcesComponents component, Repositories repository) throws Exception { + addInstances(null, component, repository, null); + for (ResourcesComponents subComponent: component.getResourcesComponents()) { + checkComponentRepoMismatches(subComponent, repository); + } + + } + /** * Method to copy resource records from one database to the next * @@ -1611,28 +1704,16 @@ public void copyResourceRecords(int max, int threads) throws Exception { currentRecordType = "Resource Record"; - // first delete previously saved resource records if that option was selected by user -// if(deleteSavedResources) { -// deleteSavedResources(); -// } - if (!recordsToCopy.contains("Resource Records")) return; ArrayList records = sourceRCD.getResources(); -// if (recordsToCopy.peek().equals("Resource Records")) { -// records = new ArrayList(records.subList(numAttempted, records.size())); -// } - -// print("Copying " + records.size() + " Resource records ..."); - - copyCount = 0;//numSuccessful; // keep track of the number of resource records copied + // keep track of the number of resource records copied + copyCount = 0; // these are used to update the progress bar int total = records.size(); - int count = 0;//numAttempted; - -// records = new ArrayList(records.subList(numAttempted, total)); + int count = 0; print("Copying " + records.size() + " Resource records ..."); @@ -1645,6 +1726,8 @@ public void copyResourceRecords(int max, int threads) throws Exception { count++; + TopContainerMapper.resetAlreadyAdded(resource.getIdentifier()); + // check if to stop copy process if(stopCopy) { updateRecordTotals("Resource Records", total, copyCount); @@ -1739,7 +1822,7 @@ public void copyResourceRecords(int max, int threads) throws Exception { JSONObject componentJS = (JSONObject) mapper.convert(component); if (componentJS != null) { - componentJS.put("resource", mapper.getReferenceObject(resourceURI)); + componentJS.put("resource", ASpaceMapper.getReferenceObject(resourceURI)); // add the subjects now addSubjects(componentJS, component); @@ -2020,6 +2103,9 @@ private synchronized void addInstances(JSONObject json, Set)uriMap.get(LOCATION_KEY); subjectURIMap = (HashMap)uriMap.get(SUBJECT_KEY); - otherSubjectURIMap = (HashMap)uriMap.get(OTHER_SUBJECT_KEY); nameURIMap = (HashMap)uriMap.get(NAME_KEY); accessionURIMap = (HashMap)uriMap.get(ACCESSION_KEY); digitalObjectURIMap = (HashMap)uriMap.get(DIGITAL_OBJECT_KEY); @@ -2785,6 +2877,7 @@ public void loadURIMaps() { assessmentURIMap = (HashMap)uriMap.get(ASSESSMENT_KEY); repositoryURIMap = (HashMap)uriMap.get(REPOSITORY_KEY); repositoryAgentURIMap = (HashMap)uriMap.get(REPOSITORY_AGENT_KEY); + userURIMap = (HashMap)uriMap.get(USER_KEY); HashMap repoGroupMap = (HashMap)uriMap.get(REPOSITORY_GROUP_KEY); for (String key: repoGroupMap.keySet()) { diff --git a/src/org/archiviststoolkit/plugin/utils/aspace/ASpaceEnumUtil.java b/src/org/archiviststoolkit/plugin/utils/aspace/ASpaceEnumUtil.java index c28474d..917affc 100644 --- a/src/org/archiviststoolkit/plugin/utils/aspace/ASpaceEnumUtil.java +++ b/src/org/archiviststoolkit/plugin/utils/aspace/ASpaceEnumUtil.java @@ -12,10 +12,9 @@ * This is util class used to mapped ASpace enum list to AR lookup list items * * Created by IntelliJ IDEA. - * User: nathan - * Date: 12/7/12 - * Time: 9:52 AM - * Updated by sarah morrissey 9/2017 + * @author nathan + * Updated by sarah morrissey 12/2017 + * @version 2.2 */ public class ASpaceEnumUtil { private HashMap languageCodes; @@ -68,8 +67,9 @@ public void setNameLinkCreatorCodes(HashMap nameLinkCreatorCodes * @return */ public Object[] getASpaceTermType(String atValue) { - if (atValue.contains("uniform")) { - atValue = "uniform_title"; + + if (atValue.contains("Genre")) { + atValue = "genre_form"; } return getASpaceEnumValue("subject_term_type", atValue, false, "topical"); } @@ -95,15 +95,19 @@ public Object[] getASpaceSubjectSource(String atValue) { code = "tgn"; } else if (atValue.contains("library of congress subject headings")) { code = "lcsh"; - } else if (atValue.equals("local")) { + } else if (atValue.contains("local sources")) { code = "local"; } else if (atValue.contains("medical subject headings")) { code = "mesh"; } else if (atValue.contains("thesaurus for graphic materials")) { code = "gmgpc"; } else { - if (code == null || code.isEmpty()) code = "local"; - else code = code.replace(".", ""); + if (code == null) { + code = "local"; + } else { + code = code.replace(".", ""); + if (code.isEmpty()) code = "local"; + } } return getASpaceEnumValue("subject_source", code); @@ -117,7 +121,7 @@ public Object[] getASpaceSubjectSource(String atValue) { */ public Object[] getASpaceNameSource(String atValue) { - if (atValue == null || atValue.trim().isEmpty()) atValue = "local"; + if (atValue == null) atValue = ""; atValue = atValue.toLowerCase(); if(atValue.contains("naco")) { @@ -126,6 +130,8 @@ public Object[] getASpaceNameSource(String atValue) { atValue = "nad"; } else if(atValue.contains("union")) { atValue = "ulan"; + } else if (atValue.contains("local sources")) { + atValue = "local"; } return getASpaceEnumValue("name_source", atValue); } @@ -154,7 +160,8 @@ public Object[] getASpaceNameOrder(Boolean directOrder) { * @return */ public Object[] getASpaceNameRule(String atValue) { - if(atValue == null || atValue.isEmpty()) atValue = "local"; + + if (atValue == null) atValue = ""; atValue = atValue.toLowerCase(); @@ -239,9 +246,6 @@ public Object[] getASpaceDateCertainty(ArchDescriptionDates archDescriptionDate) if(archDescriptionDate != null && archDescriptionDate.getCertainty() != null && !archDescriptionDate.getCertainty()) { atValue = "approximate"; } -// } else { -// atValue = "questionable"; -// } return getASpaceEnumValue("date_certainty", atValue, false, null); } @@ -281,7 +285,7 @@ public Object[] getASpaceCollectionManagementRecordProcessingStatus(String atVal } /** - * + * gets ASpace agent role * @param atValue * @return */ @@ -472,12 +476,7 @@ public Object[] getASpaceSinglePartNoteType(String atValue) { } else if(atValue.contains("physical facet")) { atValue = "physfacet"; } - Object[] value = getASpaceEnumValue(enumName, atValue, false, defaultValue); - if (value[0].equals(defaultValue)) { - Object[] multiValue = getASpaceMultiPartNoteType(atValue); - if (!(multiValue[0] == null || multiValue[0].equals("odd"))) value = multiValue; - } - return value; + return getASpaceEnumValue(enumName, atValue, false, defaultValue); } /** @@ -620,15 +619,30 @@ public Object[] getASpaceAcquisitionType(String atValue) { * @return */ public Object[] getASpaceAccessionResourceType(String atValue) { - if(atValue == null || atValue.isEmpty()) atValue = "collection"; + if(atValue == null || atValue.trim().isEmpty()) atValue = "collection"; return getASpaceEnumValue("accession_resource_type", atValue); } + /** + * method to get the ASpace rights basis + * @param atValue + * @return + */ public Object[] getASpaceRightsBasis(String atValue) { - if (atValue == null || atValue.isEmpty()) atValue = "archivists_toolkit"; + if (atValue == null || atValue.trim().isEmpty()) atValue = "archivists_toolkit"; return getASpaceEnumValue("rights_statement_other_rights_basis", atValue); } + /** + * Method to get the ASpace country ID + * @param atValue + * @return + */ + public Object[] getASpaceCountryID(String atValue) { + if (atValue != null && atValue.trim().length() != 2) atValue = null; + return getASpaceEnumValue("country_iso_3166", atValue, false, null); + } + /** * Method to set the hash map that holds the dynamic enums * @@ -732,7 +746,7 @@ private Object[] getASpaceEnumValue(String enumListName, String atValue, boolean //this really shouldn't occur but is here as a safety measure if (dynamicEnums == null) { return new Object[]{null, false}; - } + } //if value is null go ahead and return it if (atValue == null) { @@ -743,6 +757,11 @@ private Object[] getASpaceEnumValue(String enumListName, String atValue, boolean atValue = atValue.trim().toLowerCase(); atValue = atValue.replace(" ", "_"); + if (atValue.isEmpty()) { + atValue = defaultValue; + if (defaultValue == null) return new Object[]{null, false}; + } + try { //if there is a value in ASpace that matches return this and true JSONArray enumValues = dynamicEnums.get(enumListName).getJSONArray("values"); @@ -792,8 +811,6 @@ public Object[] mapsToASpaceEnumValue(String enumListName, String atValue, Strin mappedValue = getASpaceNameSource(atValue); } else if(enumListName.equals("name_rule")) { mappedValue = getASpaceNameRule(atValue); -// } else if(enumListName.equals("name_description_type")) { -// mappedValue = getASpaceNameDescriptionType(atValue); } else if(enumListName.equals("accession_acquisition_type")) { mappedValue = getASpaceAcquisitionType(atValue); } else if(enumListName.equals("accession_resource_type")) { diff --git a/src/org/archiviststoolkit/plugin/utils/aspace/ASpaceMapper.java b/src/org/archiviststoolkit/plugin/utils/aspace/ASpaceMapper.java index 9060e58..de40e0f 100644 --- a/src/org/archiviststoolkit/plugin/utils/aspace/ASpaceMapper.java +++ b/src/org/archiviststoolkit/plugin/utils/aspace/ASpaceMapper.java @@ -14,9 +14,9 @@ /** * Created by IntelliJ IDEA. - * User: nathan - * Date: 9/5/12 - * Time: 1:41 PM + * @author nathan + * Updated by Sarah Morrissey 12/2017 + * @version 2.2 * * Class to map AT data model to ASPace JSON data model */ @@ -298,13 +298,7 @@ public String convertSubject(Subjects record) throws Exception { * @throws Exception */ public JSONObject addSubjectTerm(Subjects record, JSONObject json) throws Exception { - JSONArray termsJA; - try { - //if there is already a list with some terms start with it - termsJA = (JSONArray) json.get("terms"); - } catch (JSONException e) { - termsJA = new JSONArray(); - } + JSONArray termsJA = new JSONArray(); // see if to define term type as untyped boolean isDefault = aspaceCopyUtil.isTermTypeDefault(); @@ -476,6 +470,11 @@ private String addBasicNameInformation(JSONObject namesJS, BasicNames record) th String nameSource = (String) enumUtil.getASpaceNameSource(((Names)record).getNameSource())[0]; String nameRule = (String) enumUtil.getASpaceNameRule(((Names)record).getNameRule())[0]; + // we must have either a source or rules in ASpace so set source to unknown if both are empty in AT + if ((nameSource == null || nameSource.isEmpty()) && (nameRule == null || nameRule.isEmpty())) { + nameSource = (String) enumUtil.getASpaceNameSource("unknown")[0]; + } + namesJS.put("source", nameSource); namesJS.put("rules", nameRule); } @@ -566,15 +565,15 @@ public String getCorporateAgent(Repositories repository) throws JSONException { JSONArray contactsJA = new JSONArray(); JSONObject contactsJS = new JSONObject(); - contactsJS.put("name", repository.getShortName()); + contactsJS.put("name", fixEmptyString(repository.getRepositoryName(), repository.getShortName())); contactsJS.put("address_1", repository.getAddress1()); contactsJS.put("address_2", repository.getAddress2()); contactsJS.put("address_3", repository.getAddress3()); contactsJS.put("city", repository.getCity()); + contactsJS.put("region", repository.getRegion()); - // add the country and country code together - String country = repository.getCountry() + " " + repository.getCountryCode(); - contactsJS.put("country", country.trim()); + // add the country, country code is added to the repository itself + contactsJS.put("country", repository.getCountry().trim()); contactsJS.put("post_code", repository.getMailCode()); contactsJS.put("email", repository.getEmail()); @@ -644,6 +643,8 @@ public String convertRepository(Repositories record) throws Exception { json.put("repo_code", record.getShortName()); json.put("name", fixEmptyString(record.getRepositoryName())); json.put("org_code", record.getAgencyCode()); + String countryCode = (String) enumUtil.getASpaceCountryID(record.getCountryCode())[0]; + if (countryCode != null) json.put("country", countryCode); json.put("parent_institution_name", record.getInstitutionName()); json.put("url", fixUrl(record.getUrl())); json.put("publish", publishHashMap.get("repositories")); @@ -712,7 +713,7 @@ public String convertUser(Users record) throws Exception { json.put("username", username); // get the full name, if it doesn't exist then just enter text with random string - String name = fixEmptyString(record.getFullName(), "full name not entered ##" + randomString.nextString()); + String name = fixEmptyString(record.getFullName(), "User: " + username); json.put("name", name); json.put("email", record.getEmail()); @@ -780,9 +781,8 @@ public JSONObject convertAccession(Accessions record) throws Exception { //json.put("disposition", record.?); json.put("inventory",record.getInventory()); - //json.put("provenance",record.?); - /* add linked records (extents, dates, rights statement) */ + // add linked records (extents, dates, rights statement) // add the extent array containing one object or many depending if we using multiple extents JSONArray extentJA = new JSONArray(); @@ -860,7 +860,14 @@ public JSONObject convertAccession(Accessions record) throws Exception { json.put("retention_rule", record.getRetentionRule()); - json.put("general_note", record.getGeneralAccessionNote()); + // add info on who created the accession record to general note field + String note = ""; + note += record.getGeneralAccessionNote(); + note += "\n\nAT record created by: "; + note += record.getCreatedBy(); + note += ".\n"; + + json.put("general_note", note); json.put("access_restrictions", record.getAccessRestrictions()); @@ -1298,7 +1305,7 @@ public JSONObject convertDigitalObject(DigitalObjects record) throws Exception { json.put("dates", dateJA); } - /* add the fields required digital_object.rb */ + // add the fields required digital_object.rb JSONArray fileVersionsJA = new JSONArray(); convertFileVersions(fileVersionsJA, record.getFileVersions()); @@ -1461,7 +1468,7 @@ public JSONObject convertResource(Resources record) throws Exception { json.put("external_documents", externalDocumentsJA); } - /* Add fields needed for resource.rb */ + // Add fields needed for resource.rb // get the ids and make them unique if we in DEBUG mode String[] cleanIds = cleanUpIds(ASpaceClient.RESOURCE_ENDPOINT, record.getResourceIdentifier1().trim(), @@ -1506,7 +1513,6 @@ public JSONObject convertResource(Resources record) throws Exception { json.put("ead_location", record.getEadFaLocation()); // TODO 5/12/2015 -- breakout finding aid title and subtitle into separate fields in the ASpace record - //json.put("finding_aid_title", record.getFindingAidTitle() + "\n" + record.getFindingAidSubtitle()); json.put("finding_aid_title", record.getFindingAidTitle()); json.put("finding_aid_subtitle", record.getFindingAidSubtitle()); json.put("finding_aid_filing_title", record.getFindingAidFilingTitle()); @@ -1588,7 +1594,7 @@ private JSONObject convertResourceComponent(ResourcesComponents record) throws E json.put("title", "unspecified"); } - /* add field required for archival_object.rb */ + // add field required for archival_object.rb // see if to make the ref id unique, leave blank, or just use the original (default) String refId; @@ -1674,8 +1680,11 @@ private String convertAssessment(Assessments record) throws Exception { addExternalId(record, json, "assessment"); + // add the name for who did the survey. If empty use name of user who created record print("Adding agent record for who did survey ..."); - json.put("surveyed_by", addAssessmentsAgent(record.getWhoDidSurvey(), record)); + String name = record.getWhoDidSurvey(); + if (name == null || name.isEmpty()) name = record.getCreatedBy(); + json.put("surveyed_by", addAssessmentsAgent(name, record)); Date surveyBegin = record.getDateOfSurvey(); //use the date the assessment record was created if the date of survey is not specified @@ -1699,8 +1708,6 @@ private String convertAssessment(Assessments record) throws Exception { json.put("inactive", record.getInactive()); - addAssessmentsAttributes(json, record); - json.put("general_assessment_note", record.getGeneralNote()); json.put("special_format_note", record.getSpecialFormatNote()); json.put("exhibition_value_note", record.getExhibitionValueNote()); @@ -1715,36 +1722,48 @@ private String convertAssessment(Assessments record) throws Exception { } /** - * method to add the linked records with matching repository to an assessment + * method to add the linked records with matching repository to an assessment as well as add the assessment attributes * @param jsonText * @param record * @return * @throws Exception */ - public String addAssessmentsRecords(String jsonText, Assessments record) throws Exception { + public String addAssessmentsRepoSpecificInfo(String jsonText, Assessments record, String repoURI) throws Exception { JSONObject json = new JSONObject(jsonText); JSONArray recordsJA = new JSONArray(); String recordUri; + + // add linked accessions from the correct repository for (AssessmentsAccessions accession : record.getAccessions()) { Accessions accessionRecord = accession.getAccession(); - if (accessionRecord.getRepository().equals(record.getRepository())) { + String recordRepo = aspaceCopyUtil.getRemappedRepositoryURI("accession", + accessionRecord.getIdentifier(), accessionRecord.getRepository()); + if (recordRepo.equals(repoURI)) { recordUri = aspaceCopyUtil.getURIMapping(accessionRecord); recordsJA.put(getReferenceObject(recordUri)); } } + + // add linked digital objects from correct repository for (AssessmentsDigitalObjects digitalObject: record.getDigitalObjects()) { DigitalObjects digitalObjectRecord = digitalObject.getDigitalObject(); - if (digitalObjectRecord.getRepository().equals(record.getRepository())) { + String recordRepo = aspaceCopyUtil.getRemappedRepositoryURI("digitalObject", + digitalObjectRecord.getIdentifier(), digitalObjectRecord.getRepository()); + if (recordRepo.equals(repoURI)) { recordUri = aspaceCopyUtil.getURIMapping(digitalObjectRecord); recordsJA.put(getReferenceObject(recordUri)); } } + + // add resources from the correct repository for (AssessmentsResources resource : record.getResources()) { Resources resourceRecord = resource.getResource(); - if (resourceRecord.getRepository().equals(record.getRepository())) { + String recordRepo = aspaceCopyUtil.getRemappedRepositoryURI("resource", + resourceRecord.getIdentifier(), resourceRecord.getRepository()); + if (recordRepo.equals(repoURI)) { recordUri = aspaceCopyUtil.getURIMapping(resourceRecord); recordsJA.put(getReferenceObject(recordUri)); } @@ -1752,6 +1771,9 @@ public String addAssessmentsRecords(String jsonText, Assessments record) throws json.put("records", recordsJA); + // also add the assessment attributes since they also must be scoped by repository + addAssessmentsAttributes(json, record, repoURI); + return json.toString(); } @@ -1771,17 +1793,22 @@ private JSONArray addAssessmentsAgent(String name, Assessments assessment) throw name = name.trim(); boolean unknown = false; - if (name == null || name.isEmpty()) { + if (name.isEmpty()) { //if a default name 'unknown' has already been added use that instead of adding again if (unknownName != null) return unknownName; name = "unknown"; unknown = true; } + JSONObject matchingUserAgentRef = aspaceCopyUtil.getMatchingUserAgent(name); + if (matchingUserAgentRef != null) { + return new JSONArray().put(matchingUserAgentRef); + } + //map the name to an ASpace agent nameJSON.put("primary_name", name); nameJSON.put("sort_name", name); - nameJSON.put("source", enumUtil.getASpaceNameSource(null)[0]); + nameJSON.put("source", enumUtil.getASpaceNameSource("local")[0]); nameJSON.put("name_order", enumUtil.getASpaceNameOrder(null)[0]); namesJA.put(nameJSON); json.put("names", namesJA); @@ -1793,18 +1820,25 @@ private JSONArray addAssessmentsAgent(String name, Assessments assessment) throw String endpoint = "/agents/people"; String id = aspaceCopyUtil.saveRecord(endpoint , json.toString(), "Assessments->" + assessment.getIdentifier()); - //return a JSONArray with a reference to the agent - String uri = endpoint + "/" + id; - JSONArray ja = new JSONArray(); - ja.put(getReferenceObject(uri)); - if (unknown) unknownName = ja; + if (!id.equals("no id assigned")) { + //return a JSONArray with a reference to the agent + String uri = endpoint + "/" + id; + aspaceCopyUtil.addAssessmentsNameToUserURIMap(name, uri); + ja.put(getReferenceObject(uri)); + if (unknown) unknownName = ja; + } return ja; } - private void addAssessmentsAttributes(JSONObject json, Assessments record) throws Exception { - - Repositories repo = record.getRepository(); + /** + * add assessment attributes to an assessment + * @param json + * @param record + * @param repo + * @throws Exception + */ + private void addAssessmentsAttributes(JSONObject json, Assessments record, String repo) throws Exception { //first add the formats JSONArray formatsJA = new JSONArray(); @@ -2080,8 +2114,6 @@ public void addNote(JSONObject agentJS, Names record) throws Exception { JSONObject noteJS = new JSONObject(); noteJS.put("jsonmodel_type", "note_bioghist"); -// noteJS.put("publish", publishHashMap.get("names")); -// noteJS.put("label", enumUtil.getASpaceNameDescriptionType(record.getDescriptionType())); JSONArray subnotesJA = new JSONArray(); @@ -2160,13 +2192,13 @@ public void addNotes(JSONArray notesJA, ArchDescription record) throws Exception } else { // even though it could be a single part note, based on the type it // needs to be a multi part note in ASpace - noteType = (String) enumUtil.getASpaceSinglePartNoteType(noteType)[0]; + String noteTypeMapped = (String) enumUtil.getASpaceSinglePartNoteType(noteType)[0]; - if(noteType.equals(ASpaceEnumUtil.UNMAPPED)) { + if(noteTypeMapped.equals("abstract") && !noteType.toLowerCase().contains("abstract")) { addMultiPartNote(noteJS, note); } else { noteJS.put("jsonmodel_type", "note_singlepart"); - noteJS.put("type", noteType); + noteJS.put("type", noteTypeMapped); noteJS.put("content", contentJA); } } @@ -2401,9 +2433,7 @@ private void addIndexNote(JSONObject noteJS, Index index) throws Exception { itemJS.put("value", indexItem.getItemValue()); itemJS.put("type", enumUtil.getASpaceIndexItemType(indexItem.getItemType())[0]); itemJS.put("reference", indexItem.getReference()); - //itemJS.put("reference", fixEmptyString(indexItem.getReference(), null)); itemJS.put("reference_text", indexItem.getReferenceText()); - //itemJS.put("reference_text", fixEmptyString(indexItem.getReferenceText(), null)); itemsJA.put(itemJS); } @@ -2501,21 +2531,6 @@ public JSONObject convertAnalogInstance(ArchDescriptionAnalogInstances analogIns containerJS.put("indicator_3", indicator3); } -// // add the location now if needed -// if(locationURI != null && !locationURI.isEmpty()) { -// Date date = new Date(); // this is need to have valid container_location json record -// -// JSONArray locationsJA = new JSONArray(); -// -// JSONObject locationJS = new JSONObject(); -// locationJS.put("status", "current"); -// locationJS.put("start_date", date); -// locationJS.put("ref", locationURI); -// -// locationsJA.put(locationJS); -// containerJS.put("container_locations", locationsJA); -// } - // TODO 4/16/2013 add the user defined fields //addUserDefinedFields(containerJS, analogInstance); @@ -2543,8 +2558,7 @@ public JSONObject convertDigitalInstance(String digitalObjectURI) throws Excepti } /** - * Method to create a dummy instance to old the location information - * + * Method to create a dummy instance to hold the location information for an accession * * @param locationNote * @return @@ -2560,22 +2574,11 @@ public JSONObject createAccessionInstance(Accessions accession, String locationU // add the container now JSONObject containerJS = new JSONObject(); + // create a top container or get it if a equivalent one exists TopContainerMapper topContainer = new TopContainerMapper(location, accession, parentRepoURI); topContainer.addLocationURI(locationURI, locationNote); containerJS.put("top_container", getReferenceObject(topContainer.getRef())); -// Date date = new Date(); // this is need to have valid container_location json record -// JSONArray locationsJA = new JSONArray(); -// -// JSONObject locationJS = new JSONObject(); -// locationJS.put("status", "current"); -// locationJS.put("start_date", date); -// locationJS.put("ref", locationURI); -// locationJS.put("note", locationNote); -// -// locationsJA.put(locationJS); -// -// containerJS.put("container_locations", locationsJA); instanceJS.put("sub_container", containerJS); return instanceJS; @@ -2637,7 +2640,7 @@ public void setReturnATValue(boolean value) { * @param lookupList * @return */ - public JSONObject mapLookList(LookupList lookupList) throws Exception { + public JSONObject mapLookList(LookupList lookupList, ArrayList additional) throws Exception { // first we get the correct dynamic enum based on list. If it null then we just return null JSONObject dynamicEnumJS = enumUtil.getDynamicEnum(lookupList.getListName()); @@ -2647,19 +2650,32 @@ public JSONObject mapLookList(LookupList lookupList) throws Exception { String enumListName = dynamicEnumJS.getString("name"); JSONArray valuesJA = dynamicEnumJS.getJSONArray("values"); + HashSet values = new HashSet(); + for (LookupListItems lookupListItem: lookupList.getListItems()) { String atValue = lookupListItem.getListItem(); String code = lookupListItem.getCode(); -// String toAdd; -// if (enumListName.equals("subject_source")) toAdd = code; -// else toAdd = atValue.toLowerCase(); + // get the value it maps to and whether or not that value is already in ASpace Object[] mapped = enumUtil.mapsToASpaceEnumValue(enumListName, atValue, code); + + // if its not already in ASpace, add it if(!(Boolean) mapped[1]) { - valuesJA.put(mapped[0]); + values.add((String) mapped[0]); } } + // we may have additional items we want in ASpace as defaults that aren't in AT + // add these values as needed + for (String value: additional) { + Object[] mapped = enumUtil.mapsToASpaceEnumValue(enumListName, value, ""); + if (!(Boolean) mapped[1]) { + values.add((String) mapped[0]); + } + } + + for (String value: values) valuesJA.put(value); + return dynamicEnumJS; } @@ -2922,24 +2938,8 @@ private String getUniqueID(String endpoint, String id, String[] idParts) { // is being used to to store the new id return "not used"; } else if(endpoint.equals(ASpaceClient.RESOURCE_ENDPOINT)) { - String message = null; - //I think its fine without checking for duplicate resource IDs and this causes a problem if checked - if(true) {//!resourceIDs.contains(id)) { - resourceIDs.add(id); - } else { - String fullId = ""; - - do { - idParts[0] += " ##" + randomString.nextString(); - fullId = concatIdParts(idParts); - } while(resourceIDs.contains(fullId)); - - resourceIDs.add(fullId); - - message = "Duplicate Resource Id: " + id + " Changed to: " + fullId + "\n"; - aspaceCopyUtil.addErrorMessage(message); - } + resourceIDs.add(id); // we don't need to return the new id here, since the idParts array // is being used to to store the new id diff --git a/src/org/archiviststoolkit/plugin/utils/aspace/TopContainerMapper.java b/src/org/archiviststoolkit/plugin/utils/aspace/TopContainerMapper.java index c21cefd..00353f5 100644 --- a/src/org/archiviststoolkit/plugin/utils/aspace/TopContainerMapper.java +++ b/src/org/archiviststoolkit/plugin/utils/aspace/TopContainerMapper.java @@ -15,27 +15,37 @@ * class to add and manage top containers * * @author sarah morrissey - * date: 9/2017 + * date: 12/2017 + * @version 2.2 */ public class TopContainerMapper { - //keys are containers that have been added to ASpace and values are objects that store that container's information + // keys are containers that have been added to ASpace and values are objects that store that container's information private static HashMap alreadyAdded = new HashMap(); - //used for making unique identifiers for containers without indicators + // these are the containers with barcodes that we need to keep track of even if we are done copying the current resoruce + private static HashMap permanentAdded = new HashMap(); + + // this gets set each time we begin copying a new resource + // its -1 if copying an accession + private static long resourceID = -1; + + // used for making unique identifiers for containers without indicators private static int unknownCount = 1; private ASpaceEnumUtil enumUtil = new ASpaceEnumUtil(); private static ASpaceCopyUtil aSpaceCopyUtil; - //analog instance or accession location the container was created for + // analog instance or accession location the container was created for private DomainObject instance; + private Accessions accession; + + // the stuff that is needed to save the container private String atID; private String parentRepoURI; private String indicator; private String type; private String barcode; - private Accessions accession; public DomainObject getInstance() { return instance; @@ -183,7 +193,10 @@ private void saveTopContainerLocation(String uri, JSONObject location) throws Ex */ public void addToExisting(String uri) { if (uri.contains("10000001") || uri.contains("no id assigned")) return; - alreadyAdded.put(new MiniContainer(this), new Info(uri)); + MiniContainer mini = new MiniContainer(this); + Info info = new Info(uri); + alreadyAdded.put(mini, info); + if (this.barcode != null && !this.barcode.isEmpty()) permanentAdded.put(mini, info); } /** @@ -269,29 +282,40 @@ private static class MiniContainer { private String indicator; private String type; private String barcode; + private long resourceID; + /** + * constructs a mini container version of a top container object + * @param container the top container we are creating the mini container for + */ MiniContainer(TopContainerMapper container) { this.parentRepoURI = container.parentRepoURI; this.indicator = container.indicator; this.type = container.type; this.barcode = container.barcode; + this.resourceID = TopContainerMapper.resourceID; } + /** + * reconstructs a mini container from a saved string + * @param args the string splint into its components + */ MiniContainer(String ... args) { this.parentRepoURI = args[0]; - this.indicator = args[1]; - if (args.length > 2) { - if (args[2].contains("type: ")) { - this.type = args[2].substring(6); - if (args.length > 3) this.barcode = args[3]; + this.resourceID = Long.parseLong(args[1]); + this.indicator = args[2]; + if (args.length > 3) { + if (args[3].contains("type: ")) { + this.type = args[3].substring(6); + if (args.length > 4) this.barcode = args[4]; } - else this.barcode = args[2]; + else this.barcode = args[3]; } } @Override public String toString() { - return parentRepoURI + SEPARATOR + indicator + SEPARATOR + "type: " + type + SEPARATOR + barcode; + return parentRepoURI + SEPARATOR + resourceID + SEPARATOR + indicator + SEPARATOR + "type: " + type + SEPARATOR + barcode; } @Override @@ -299,12 +323,21 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || !(o instanceof MiniContainer)) return false; MiniContainer other = (MiniContainer) o; + + // records from different repositories can not be in the same top container if (!(this.parentRepoURI.equals(other.parentRepoURI))) return false; + + // if two containers have the same non-empty barcode, they are the same if (!(this.barcode == null || this.barcode.isEmpty() || other.barcode == null || other.barcode.isEmpty())) { if (this.barcode.trim().equalsIgnoreCase(other.barcode.trim())) return true; } + + // unless barcode shows otherwise, different resources are assumed to be in different containers + if (this.resourceID != other.resourceID) return false; + + // finally if they have the same type and indicator, they are the same container Boolean sameType = true; - if (!(this.type == null || this.type.isEmpty() || other.type == null || other.type.isEmpty())) { + if (!(this.type == null || other.type == null)) { sameType = this.type.equals(other.type); } return (this.indicator.equalsIgnoreCase(other.indicator) && sameType); @@ -312,6 +345,7 @@ public boolean equals(Object o) { @Override public int hashCode() { + // can't do too much here - repository is the only thing guaranteed to be the same for equivalent containers return parentRepoURI.hashCode(); } } @@ -327,6 +361,10 @@ public Info(String uri) { this.uri = uri; } + /** + * recreates the info from a string + * @param args string split into its components + */ public Info(String ... args) { this.uri = args[0]; for (int i = 1; i < args.length; i++) locationURIs.add(args[i]); @@ -343,12 +381,22 @@ public String toString() { } } + /** + * splits a saved string for a mini container or info object into its components + * @param stringForm the saved string + * @return the saved string split according to the separator + */ private static String[] fromString(String stringForm) { return stringForm.split(SEPARATOR); } + // the separator that is used when making string forms of thing for saving private static final String SEPARATOR = "! . . . !"; + /** + * turns the already added map into a string to string map that can be saved + * @return a string to string hashmap that can be saved to a file + */ public static HashMap getAlreadyAddedStringForm() { HashMap alreadyAddedStringForm = new HashMap(); for (MiniContainer container : alreadyAdded.keySet()) { @@ -357,15 +405,34 @@ public static HashMap getAlreadyAddedStringForm() { return alreadyAddedStringForm; } + /** + * take a saved string to string map and processes the data to make the already added map + * @param topContainerURIMap the string to string version of already added + */ public static void setAlreadyAdded(HashMap topContainerURIMap) { for (String key: topContainerURIMap.keySet()) { MiniContainer miniContainer = new MiniContainer(fromString(key)); Info info = new Info(fromString(topContainerURIMap.get(key))); alreadyAdded.put(miniContainer, info); + if (miniContainer.barcode != null && !miniContainer.barcode.isEmpty()) permanentAdded.put(miniContainer, info); } } + /** + * clears everything from the list of containers that have been added + */ public static void clearAlreadyAdded() { + permanentAdded = new HashMap(); + resetAlreadyAdded(-1); + } + + /** + * clears all the containers that do not have barcodes from already added and sets the current resource ID + * @param resourceID the resource that is being copied now + */ + public static void resetAlreadyAdded(long resourceID) { + TopContainerMapper.resourceID = resourceID; alreadyAdded = new HashMap(); + alreadyAdded.putAll(permanentAdded); } } diff --git a/src/test/ASpaceEnumUtilTest.java b/src/test/ASpaceEnumUtilTest.java index 387205c..54f478d 100644 --- a/src/test/ASpaceEnumUtilTest.java +++ b/src/test/ASpaceEnumUtilTest.java @@ -1,6 +1,10 @@ package test; import junit.framework.JUnit4TestAdapter; +import org.archiviststoolkit.model.LookupList; +import org.archiviststoolkit.model.LookupListItems; +import org.archiviststoolkit.plugin.dbdialog.RemoteDBConnectDialogLight; +import org.archiviststoolkit.plugin.utils.aspace.ASpaceClient; import org.archiviststoolkit.plugin.utils.aspace.ASpaceEnumUtil; import org.json.JSONArray; import org.json.JSONException; @@ -8,6 +12,9 @@ import org.junit.Assert; import org.junit.Test; +import java.util.ArrayList; +import java.util.HashMap; + public class ASpaceEnumUtilTest extends Testing { static JSONObject json = new JSONObject(); @@ -32,6 +39,63 @@ public void emptyMapsToLocal() throws JSONException { Assert.assertEquals("local", result[0]); } + /** + * this is an easy way to get all the lookup list mappings for the data map spreadsheet + * will need to modify connection information before using + * @throws JSONException + */ +// @Test +// public void mapAllLookupListItems() throws JSONException { +// ASpaceClient asc = new ASpaceClient("http://localhost:8089", "admin", "admin"); +// ASpaceEnumUtil tester = new ASpaceEnumUtil(); +// HashMap dynamicEnums = asc.loadDynamicEnums(); +// tester.setASpaceDynamicEnums(dynamicEnums); +// RemoteDBConnectDialogLight rcd = new RemoteDBConnectDialogLight(); +// rcd.connectToDatabase("MySQL", "jdbc:mysql://localhost:3306/at", "sem", "sem"); +// ArrayList lookupLists = rcd.getLookupLists(); +// for (LookupList lookupList : lookupLists) { +// String listName = lookupList.getListName(); +// ArrayList additional = new ArrayList(); +// +// JSONObject dynamicEnum = tester.getDynamicEnum(listName); +// if (dynamicEnum == null) continue; +// String enumName = dynamicEnum.getString("name"); +// JSONArray valuesJA = dynamicEnum.getJSONArray("values"); +// if(listName.equalsIgnoreCase("Extent type")) { +// valuesJA.put("unknown"); +// } else if(listName.equals("Name source")) { +// valuesJA.put("unknown"); +// } else if(listName.equals("Container types")) { +// valuesJA.put("unknown_item"); +// } else if(listName.equals("Resource type")) { +// valuesJA.put("collection"); +// } else if(listName.equals("Date type")) { +// valuesJA.put("other"); +// } +// dynamicEnum.put("values", valuesJA); +// for (LookupListItems item :lookupList.getListItems()) { +// additional.add(new String[]{item.getListItem(), item.getCode()}); +// } +// additional.add(new String[]{"", null}); +// additional.add(new String[]{"other value", null}); +// +// System.out.println("\n\nAT LookupList," + listName + "\nASpace Enum," + enumName + +// "\n\nAT Value,ASpace Code,ASpace Value,Notes"); +// +// for (String[] item : additional) { +// if (item[1] == null) item[1] = ""; +// Object[] value = tester.mapsToASpaceEnumValue(enumName, item[0], item[1]); +// if (item[0].equals("")) { +// item[0] = "null/empty"; +// if (value[0] == null) continue; +// } +// System.out.println(item[0] + "," + value[0] + "," + value[1]); +// } +// } +// +// +// } + public static junit.framework.Test suite() { return new JUnit4TestAdapter(ASpaceEnumUtilTest.class); }