From 93825036844028aec7eea59978fe6dd99406a524 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Dutoit?= Date: Fri, 15 Mar 2024 09:30:19 +0100 Subject: [PATCH] Duplicates deletion --- .../ch/mno/copper/store/MapValuesStore.java | 5 ++ .../java/ch/mno/copper/store/ValuesStore.java | 2 + .../java/ch/mno/copper/store/db/DBServer.java | 48 ++++++++++++++++++- .../ch/mno/copper/store/db/DBValuesStore.java | 7 ++- .../mno/copper/web/CopperAdminServices.java | 6 +++ .../static/admin/app/controllers/values.js | 10 ++++ .../static/admin/app/views/values.html | 1 + .../ch/mno/copper/store/db/DBServerTest.java | 16 +++++++ .../ch/mno/copper/stories/data/StoryTest.java | 5 ++ 9 files changed, 97 insertions(+), 3 deletions(-) diff --git a/src/main/java/ch/mno/copper/store/MapValuesStore.java b/src/main/java/ch/mno/copper/store/MapValuesStore.java index ba5ac30..41d1625 100644 --- a/src/main/java/ch/mno/copper/store/MapValuesStore.java +++ b/src/main/java/ch/mno/copper/store/MapValuesStore.java @@ -89,6 +89,11 @@ public String deleteValuesOfKey(String key) { return ""; } + @Override + public String deleteDuplicates() { + return "0"; // No historization-> no duplicates + } + /** Values as string for tests */ public String getValuesAsString() { var sb = new StringBuilder(); diff --git a/src/main/java/ch/mno/copper/store/ValuesStore.java b/src/main/java/ch/mno/copper/store/ValuesStore.java index 16faf78..4b68d5e 100644 --- a/src/main/java/ch/mno/copper/store/ValuesStore.java +++ b/src/main/java/ch/mno/copper/store/ValuesStore.java @@ -47,4 +47,6 @@ public interface ValuesStore { String deleteValuesOlderThanXDays(int nbDays); String deleteValuesOfKey(String key); + + String deleteDuplicates(); } diff --git a/src/main/java/ch/mno/copper/store/db/DBServer.java b/src/main/java/ch/mno/copper/store/db/DBServer.java index 732d7e0..06f30cc 100644 --- a/src/main/java/ch/mno/copper/store/db/DBServer.java +++ b/src/main/java/ch/mno/copper/store/db/DBServer.java @@ -78,7 +78,7 @@ public void insert(String key, String value, Instant instant) { StoreValue previousValue = readLatest(key); if (value == null) value = ""; - if (previousValue==null) { + if (previousValue == null) { insertNew(key, value, instant, stmt); } else if (!previousValue.getValue().equals(value)) { if (previousValue.getTimestampFrom().isAfter(instant)) { @@ -108,6 +108,27 @@ private void insertNew(String key, String value, Instant instant, PreparedStatem } } + void insertForTests(String key, String value, Instant dateFrom, Instant dateTo, Instant dateLastCheck){ + var sqlInsert = "INSERT INTO valuestore ( idvaluestore, vkey, vvalue, datefrom, dateto, datelastcheck) VALUES (?,?,?,?,?,?)"; + + try (var con = cp.getConnection(); + PreparedStatement stmt = con.prepareStatement(sqlInsert)) { + long id = nextSequence(); + stmt.setLong(1, id); + stmt.setString(2, key); + stmt.setString(3, value); + stmt.setTimestamp(4, Timestamp.from(dateFrom)); + stmt.setTimestamp(5, Timestamp.from(dateTo)); + stmt.setTimestamp(6, Timestamp.from(dateLastCheck)); + int rowInserted = stmt.executeUpdate(); + if (rowInserted != 1) { + throw new StoreException("DB error: inserted " + rowInserted + " values."); + } + } catch (SQLException e) { + throw new StoreException(AN_ERROR_OCCURED_WHILE_SAVING_VALUES, e); + } + } + private void terminatePrevious(Instant instant, Connection con, StoreValue previousValue) { var sqlUpdatePrevious = "update valuestore set dateto=? where idvaluestore=?"; try (PreparedStatement stmt2 = con.prepareStatement(sqlUpdatePrevious)) { @@ -164,7 +185,9 @@ public StoreValue read(String key, Instant timestamp) { } - /** Query some values at some interval of time, to plot graph */ + /** + * Query some values at some interval of time, to plot graph + */ public List readInstant(List keys, Instant timestampFrom, Instant timestampTo, long intervalSeconds, int maxValues) { String sql = "select * from (" + "select ts,c1, vvalue, idValueStore, vkey from ( " + @@ -262,6 +285,7 @@ public StoreValue readLatest(String key) { throw new StoreException(AN_ERROR_OCCURED_WHILE_SAVING_VALUES, e); } } + /** * Read all values of a key) */ @@ -314,6 +338,7 @@ public String findAlerts() { var sb = new StringBuilder(); try (var con = cp.getConnection(); PreparedStatement stmt = con.prepareStatement(sql)) { + try (var rs = stmt.executeQuery()) { while (rs.next()) { sb.append(rs.getString("vkey")); @@ -374,4 +399,23 @@ public int deleteValuesOfKey(String key) { throw new StoreException(AN_ERROR_OCCURED_WHILE_READING_VALUES, e); } } + + public int deleteDuplicates() { + try (var con = cp.getConnection(); + var stmt = con.prepareStatement("DELETE FROM valuestore\n" + + " WHERE idvaluestore NOT IN (\n" + + " SELECT idvaluesstore\n" + + " FROM (\n" + + " SELECT MAX(idvaluestore) AS idvaluesstore\n" + + " FROM valuestore\n" + + " WHERE dateto = '3000-12-31 01:00:00.000000'\n" + + " GROUP BY vkey\n" + + " ) AS subquery\n" + + ") AND dateto = ?")) { + stmt.setTimestamp(1, Timestamp.from(INSTANT_MAX)); + return stmt.executeUpdate(); + } catch (SQLException e) { + throw new StoreException(AN_ERROR_OCCURED_WHILE_READING_VALUES, e); + } + } } diff --git a/src/main/java/ch/mno/copper/store/db/DBValuesStore.java b/src/main/java/ch/mno/copper/store/db/DBValuesStore.java index d7e6b63..1e3a4aa 100644 --- a/src/main/java/ch/mno/copper/store/db/DBValuesStore.java +++ b/src/main/java/ch/mno/copper/store/db/DBValuesStore.java @@ -93,7 +93,6 @@ public Map getValuesMapString() { return server.readLatest().stream() .distinct() .collect(Collectors.toMap(StoreValue::getKey, StoreValue::getValue)); - } @Override @@ -113,4 +112,10 @@ public String deleteValuesOfKey(String key) { return "OK, " + nb + " deleted"; } + @Override + public String deleteDuplicates() { + int nb = server.deleteDuplicates(); + return "OK, " + nb + " deleted"; + } + } diff --git a/src/main/java/ch/mno/copper/web/CopperAdminServices.java b/src/main/java/ch/mno/copper/web/CopperAdminServices.java index f7c82c1..fdf95a7 100644 --- a/src/main/java/ch/mno/copper/web/CopperAdminServices.java +++ b/src/main/java/ch/mno/copper/web/CopperAdminServices.java @@ -163,5 +163,11 @@ private Gson buildGson() { return builder.create(); } + @DeleteMapping(value = "values/duplicates", produces = MediaType.TEXT_PLAIN) + @Operation(summary = "Delete duplicates, keep latest") + public String deleteDeuplicates() { + return valuesStore.deleteDuplicates(); + } + } \ No newline at end of file diff --git a/src/main/resources/static/admin/app/controllers/values.js b/src/main/resources/static/admin/app/controllers/values.js index 625b863..de504fd 100644 --- a/src/main/resources/static/admin/app/controllers/values.js +++ b/src/main/resources/static/admin/app/controllers/values.js @@ -83,6 +83,16 @@ angular.module('copperApp.values', ['ngRoute']) } } + $scope.deleteDuplicates = function(key) { + if ( window.confirm("Delete duplicates ?") ) { + $http.delete('../ws/admin/values/duplicates') + .then(function(response) { + alert(response.data); + $scope.refresh(); + }); + } + } + $scope.filterOLD = function(object, field, filter) { diff --git a/src/main/resources/static/admin/app/views/values.html b/src/main/resources/static/admin/app/views/values.html index 0efbc46..af7d05b 100644 --- a/src/main/resources/static/admin/app/views/values.html +++ b/src/main/resources/static/admin/app/views/values.html @@ -54,6 +54,7 @@

Set value

Admin

+
diff --git a/src/test/java/ch/mno/copper/store/db/DBServerTest.java b/src/test/java/ch/mno/copper/store/db/DBServerTest.java index 46075c0..447d15a 100644 --- a/src/test/java/ch/mno/copper/store/db/DBServerTest.java +++ b/src/test/java/ch/mno/copper/store/db/DBServerTest.java @@ -242,6 +242,22 @@ void testDateLastCheck() { server.readInstant(List.of("k1"), i1, i3, 1, 10).toString()); } + @Test + void testDelete() { + server.clearAllData(); + Instant i1 = Instant.parse("2021-09-30T16:42:00.00Z"); + Instant i2 = Instant.parse("2021-09-30T16:42:03.00Z"); + server.insertForTests("K1", "V1", i1, DBServer.INSTANT_MAX, i1); + server.insertForTests("K2", "V2", i1, DBServer.INSTANT_MAX, i1); + server.insertForTests("K2", "V3", i1, DBServer.INSTANT_MAX, i2); + assertEquals(1, server.readAll("K1").size()); + assertEquals(2, server.readAll("K2").size()); + + server.deleteDuplicates(); + assertEquals(1, server.readAll("K1").size()); + assertEquals(1, server.readAll("K2").size()); + } + private void assertOneValue(List values, String value) { assertNotNull(values); diff --git a/src/test/java/ch/mno/copper/stories/data/StoryTest.java b/src/test/java/ch/mno/copper/stories/data/StoryTest.java index ed3fe08..ccefd8b 100644 --- a/src/test/java/ch/mno/copper/stories/data/StoryTest.java +++ b/src/test/java/ch/mno/copper/stories/data/StoryTest.java @@ -225,6 +225,11 @@ public String deleteValuesOlderThanXDays(int nbDays) { public String deleteValuesOfKey(String key) { return null; } + + @Override + public String deleteDuplicates() { + return null; + } }; return store; }