Duplicates deletion

src/main/java/ch/mno/copper/store/MapValuesStore.java

@@ -89,6 +89,11 @@ public String deleteValuesOfKey(String key) {
         return "";
     }
 
+    @Override
+    public String deleteDuplicates() {
+        return "0"; // No historization-> no duplicates
+    }
+
     /** Values as string for tests */
     public String getValuesAsString() {
         var sb = new StringBuilder();

src/main/java/ch/mno/copper/store/ValuesStore.java

@@ -47,4 +47,6 @@ public interface ValuesStore {
     String deleteValuesOlderThanXDays(int nbDays);
 
     String deleteValuesOfKey(String key);
+
+    String deleteDuplicates();
 }

src/main/java/ch/mno/copper/store/db/DBServer.java

@@ -78,7 +78,7 @@ public void insert(String key, String value, Instant instant) {
             StoreValue previousValue = readLatest(key);
             if (value == null) value = "";
 
-            if (previousValue==null) {
+            if (previousValue == null) {
                 insertNew(key, value, instant, stmt);
             } else if (!previousValue.getValue().equals(value)) {
                 if (previousValue.getTimestampFrom().isAfter(instant)) {
@@ -108,6 +108,27 @@ private void insertNew(String key, String value, Instant instant, PreparedStatem
         }
     }
 
+    void insertForTests(String key, String value, Instant dateFrom, Instant dateTo, Instant dateLastCheck){
+        var sqlInsert = "INSERT INTO valuestore ( idvaluestore, vkey, vvalue, datefrom, dateto, datelastcheck) VALUES (?,?,?,?,?,?)";
+
+        try (var con = cp.getConnection();
+             PreparedStatement stmt = con.prepareStatement(sqlInsert)) {
+            long id = nextSequence();
+            stmt.setLong(1, id);
+            stmt.setString(2, key);
+            stmt.setString(3, value);
+            stmt.setTimestamp(4, Timestamp.from(dateFrom));
+            stmt.setTimestamp(5, Timestamp.from(dateTo));
+            stmt.setTimestamp(6, Timestamp.from(dateLastCheck));
+            int rowInserted = stmt.executeUpdate();
+            if (rowInserted != 1) {
+                throw new StoreException("DB error: inserted " + rowInserted + " values.");
+            }
+        } catch (SQLException e) {
+            throw new StoreException(AN_ERROR_OCCURED_WHILE_SAVING_VALUES, e);
+        }
+    }
+
     private void terminatePrevious(Instant instant, Connection con, StoreValue previousValue) {
         var sqlUpdatePrevious = "update valuestore set dateto=? where idvaluestore=?";
         try (PreparedStatement stmt2 = con.prepareStatement(sqlUpdatePrevious)) {
@@ -164,7 +185,9 @@ public StoreValue read(String key, Instant timestamp) {
     }
 
 
-    /** Query some values at some interval of time, to plot graph */
+    /**
+     * Query some values at some interval of time, to plot graph
+     */
     public List<InstantValues> readInstant(List<String> keys, Instant timestampFrom, Instant timestampTo, long intervalSeconds, int maxValues) {
         String sql = "select * from (" +
                 "select ts,c1, vvalue, idValueStore, vkey from ( " +
@@ -262,6 +285,7 @@ public StoreValue readLatest(String key) {
             throw new StoreException(AN_ERROR_OCCURED_WHILE_SAVING_VALUES, e);
         }
     }
+
     /**
      * Read all values of a key)
      */
@@ -314,6 +338,7 @@ public String findAlerts() {
         var sb = new StringBuilder();
         try (var con = cp.getConnection();
              PreparedStatement stmt = con.prepareStatement(sql)) {
+
             try (var rs = stmt.executeQuery()) {
                 while (rs.next()) {
                     sb.append(rs.getString("vkey"));
@@ -374,4 +399,23 @@ public int deleteValuesOfKey(String key) {
             throw new StoreException(AN_ERROR_OCCURED_WHILE_READING_VALUES, e);
         }
     }
+
+    public int deleteDuplicates() {
+        try (var con = cp.getConnection();
+             var stmt = con.prepareStatement("DELETE FROM valuestore\n" +
+                     "    WHERE idvaluestore NOT IN (\n" +
+                     "    SELECT idvaluesstore\n" +
+                     "    FROM (\n" +
+                     "        SELECT MAX(idvaluestore) AS idvaluesstore\n" +
+                     "        FROM valuestore\n" +
+                     "        WHERE dateto = '3000-12-31 01:00:00.000000'\n" +
+                     "        GROUP BY vkey\n" +
+                     "    ) AS subquery\n" +
+                     ") AND dateto = ?")) {
+            stmt.setTimestamp(1, Timestamp.from(INSTANT_MAX));
+            return stmt.executeUpdate();
+        } catch (SQLException e) {
+            throw new StoreException(AN_ERROR_OCCURED_WHILE_READING_VALUES, e);
+        }
+    }
 }

src/main/java/ch/mno/copper/store/db/DBValuesStore.java

@@ -93,7 +93,6 @@ public Map<String, String> getValuesMapString() {
         return server.readLatest().stream()
                 .distinct()
                 .collect(Collectors.toMap(StoreValue::getKey, StoreValue::getValue));
-
     }
 
     @Override
@@ -113,4 +112,10 @@ public String deleteValuesOfKey(String key) {
         return "OK, " + nb + " deleted";
     }
 
+    @Override
+    public String deleteDuplicates() {
+        int nb = server.deleteDuplicates();
+        return "OK, " + nb + " deleted";
+    }
+
 }

src/main/java/ch/mno/copper/web/CopperAdminServices.java

@@ -163,5 +163,11 @@ private Gson buildGson() {
         return builder.create();
     }
 
+    @DeleteMapping(value = "values/duplicates", produces = MediaType.TEXT_PLAIN)
+    @Operation(summary = "Delete duplicates, keep latest")
+    public String deleteDeuplicates() {
+        return valuesStore.deleteDuplicates();
+    }
+
 
 }

src/main/resources/static/admin/app/controllers/values.js

@@ -83,6 +83,16 @@ angular.module('copperApp.values', ['ngRoute'])
         }
     }
 
+    $scope.deleteDuplicates = function(key) {
+        if ( window.confirm("Delete duplicates ?") ) {
+            $http.delete('../ws/admin/values/duplicates')
+                .then(function(response) {
+                    alert(response.data);
+                    $scope.refresh();
+                });
+        }
+    }
+
 
 
     $scope.filterOLD = function(object, field, filter) {

src/main/resources/static/admin/app/views/values.html

@@ -54,6 +54,7 @@ <h2>Set value</h2>
         <h2>Admin</h2>
         <button class="btn btn-primary" ng-click="deleteValuesOlderThanOneMonth()">Delete values older than 1 month</button>
         <button class="btn btn-primary" ng-click="deleteValuesOlderThanThreeMonth()">Delete values older than 3 month</button>
+        <button class="btn btn-primary" ng-click="deleteDuplicates()">Delete duplicates</button>
     </div>
     <br/>
 

src/test/java/ch/mno/copper/store/db/DBServerTest.java

@@ -242,6 +242,22 @@ void testDateLastCheck() {
                 server.readInstant(List.of("k1"), i1, i3, 1, 10).toString());
     }
 
+    @Test
+    void testDelete() {
+        server.clearAllData();
+        Instant i1 = Instant.parse("2021-09-30T16:42:00.00Z");
+        Instant i2 = Instant.parse("2021-09-30T16:42:03.00Z");
+        server.insertForTests("K1", "V1", i1, DBServer.INSTANT_MAX, i1);
+        server.insertForTests("K2", "V2", i1, DBServer.INSTANT_MAX, i1);
+        server.insertForTests("K2", "V3", i1, DBServer.INSTANT_MAX, i2);
+        assertEquals(1, server.readAll("K1").size());
+        assertEquals(2, server.readAll("K2").size());
+
+        server.deleteDuplicates();
+        assertEquals(1, server.readAll("K1").size());
+        assertEquals(1, server.readAll("K2").size());
+    }
+
 
     private void assertOneValue(List<StoreValue> values, String value) {
         assertNotNull(values);

src/test/java/ch/mno/copper/stories/data/StoryTest.java

@@ -225,6 +225,11 @@ public String deleteValuesOlderThanXDays(int nbDays) {
             public String deleteValuesOfKey(String key) {
                 return null;
             }
+
+            @Override
+            public String deleteDuplicates() {
+                return null;
+            }
         };
         return store;
     }