Fix: Improve Milestone Feed consistency (#854)

* Fix missed milestone index. Signed-off-by: Eugene Panteleymonchuk <[email protected]> * Fix reset time when switch network. Signed-off-by: Eugene Panteleymonchuk <[email protected]> * Fix comments. Signed-off-by: Eugene Panteleymonchuk <[email protected]> --------- Signed-off-by: Eugene Panteleymonchuk <[email protected]>
iotaledger · Dec 5, 2023 · ff346f2 · ff346f2
1 parent ae687b8
commit ff346f2
Show file tree

Hide file tree

Showing 5 changed files with 99 additions and 50 deletions.
diff --git a/api/src/routes/stardust/milestone/influx/get.ts b/api/src/routes/stardust/milestone/influx/get.ts
@@ -36,7 +36,12 @@ export async function get(
     }
 
     const milestoneIndex = Number.parseInt(request.milestoneIndex, 10);
-    const maybeMsStats = await influxService.fetchAnalyticsForMilestoneWithRetries(milestoneIndex);
+    let maybeMsStats = await influxService.fetchAnalyticsForMilestoneWithRetries(milestoneIndex);
+
+    if (!maybeMsStats) {
+        maybeMsStats = await influxService.fetchAnalyticsForMilestone(milestoneIndex);
+    }
+
 
     return maybeMsStats ? {
         milestoneIndex: maybeMsStats.milestoneIndex,

diff --git a/api/src/services/stardust/influx/influxDbClient.ts b/api/src/services/stardust/influx/influxDbClient.ts
@@ -15,7 +15,8 @@ import {
     NFT_STAT_TOTAL_QUERY,
     SHIMMER_CLAIMED_TOTAL_QUERY,
     MILESTONE_STATS_QUERY,
-    STORAGE_DEPOSIT_TOTAL_QUERY
+    STORAGE_DEPOSIT_TOTAL_QUERY,
+    MILESTONE_STATS_QUERY_BY_INDEX
 } from "./influxQueries";
 import logger from "../../../logger";
 import { INetwork } from "../../../models/db/INetwork";
@@ -32,6 +33,15 @@ import {
     IUnclaimedGenesisOutputsDailyInflux, IUnclaimedTokensDailyInflux, IUnlockConditionsPerTypeDailyInflux
 } from "../../../models/influx/IInfluxTimedEntries";
 
+type MilestoneUpdate = ITimedEntry & {
+    milestoneIndex: number;
+    taggedData: number;
+    milestone: number;
+    transaction: number;
+    treasuryTransaction: number;
+    noPayload: number;
+};
+
 /**
  * The collect graph data interval cron expression.
  * Every hour at 59 min 55 sec
@@ -147,6 +157,23 @@ export abstract class InfluxDbClient {
         return false;
     }
 
+    /**
+     * Get the milestone analytics by index and set it in the cache.
+     * @param milestoneIndex - The milestone index.
+     */
+    public async collectMilestoneStatsByIndex(milestoneIndex: number) {
+        console.log('--- request ', milestoneIndex);
+        try {
+            for (const update of await
+                this._client.query<MilestoneUpdate>(MILESTONE_STATS_QUERY_BY_INDEX, { placeholders: { milestoneIndex } })
+                ) {
+                this.updateMilestoneCache(update);
+            }
+        } catch (err) {
+            logger.warn(`[InfluxDb] Failed refreshing milestone stats for "${this._network.network}". Cause: ${err}`);
+        }
+    }
+
     /**
      * Function to sort map entries in ascending order.
      * @param a The first entry
@@ -187,7 +214,7 @@ export abstract class InfluxDbClient {
                 void this.collectAnalytics();
             });
 
-            cron.schedule("*/5 * * * * *", async () => {
+            cron.schedule("*/4 * * * * *", async () => {
                 // eslint-disable-next-line no-void
                 void this.collectMilestoneStats();
             });
@@ -349,60 +376,56 @@ export abstract class InfluxDbClient {
         logger.debug(`[InfluxDb] Collecting milestone stats for "${this._network.network}"`);
         try {
             for (const update of await
-                this.queryInflux<ITimedEntry & {
-                    milestoneIndex: number;
-                    taggedData: number;
-                    milestone: number;
-                    transaction: number;
-                    treasuryTransaction: number;
-                    noPayload: number;
-                }>(
+                this.queryInflux<MilestoneUpdate>(
                     MILESTONE_STATS_QUERY, null, this.getToNanoDate()
-                )
-            ) {
-                if (update.milestoneIndex !== undefined && !this._milestoneCache.has(update.milestoneIndex)) {
-                    const {
-                        milestoneIndex, transaction, milestone, taggedData, treasuryTransaction, noPayload
-                    } = update;
-                    const blockCount = transaction + milestone + taggedData + treasuryTransaction + noPayload;
-                    this._milestoneCache.set(milestoneIndex, {
-                        milestoneIndex,
-                        blockCount,
-                        perPayloadType: {
-                            transaction,
-                            milestone,
-                            taggedData,
-                            treasuryTransaction,
-                            noPayload
-                        }
-                    });
-
-                    logger.debug(
-                        `[InfluxDb] Added milestone index "${milestoneIndex}" to cache for "${this._network.network}"`
-                    );
+                )) {
+                this.updateMilestoneCache(update);
+            }
+        } catch (err) {
+            logger.warn(`[InfluxDb] Failed refreshing milestone stats for "${this._network.network}". Cause: ${err}`);
+        }
+    }
 
-                    if (this._milestoneCache.size > MILESTONE_CACHE_MAX) {
-                        let lowestIndex: number;
-                        for (const index of this._milestoneCache.keys()) {
-                            if (!lowestIndex) {
-                                lowestIndex = index;
-                            }
+    private updateMilestoneCache(update: MilestoneUpdate) {
+        if (update.milestoneIndex !== undefined && !this._milestoneCache.has(update.milestoneIndex)) {
+            const {
+                milestoneIndex, transaction, milestone, taggedData, treasuryTransaction, noPayload
+            } = update;
+            const blockCount = transaction + milestone + taggedData + treasuryTransaction + noPayload;
+            this._milestoneCache.set(milestoneIndex, {
+                milestoneIndex,
+                blockCount,
+                perPayloadType: {
+                    transaction,
+                    milestone,
+                    taggedData,
+                    treasuryTransaction,
+                    noPayload
+                }
+            });
 
-                            if (milestoneIndex < lowestIndex) {
-                                lowestIndex = index;
-                            }
-                        }
+            logger.debug(
+                `[InfluxDb] Added milestone index "${milestoneIndex}" to cache for "${this._network.network}"`
+            );
 
-                        logger.debug(
-                            `[InfluxDb] Deleting milestone index "${lowestIndex}" ("${this._network.network}")`
-                        );
+            if (this._milestoneCache.size > MILESTONE_CACHE_MAX) {
+                let lowestIndex: number;
+                for (const index of this._milestoneCache.keys()) {
+                    if (!lowestIndex) {
+                        lowestIndex = index;
+                    }
 
-                        this._milestoneCache.delete(lowestIndex);
+                    if (milestoneIndex < lowestIndex) {
+                        lowestIndex = index;
                     }
                 }
+
+                logger.debug(
+                    `[InfluxDb] Deleting milestone index "${lowestIndex}" ("${this._network.network}")`
+                );
+
+                this._milestoneCache.delete(lowestIndex);
             }
-        } catch (err) {
-            logger.warn(`[InfluxDb] Failed refreshing milestone stats for "${this._network.network}". Cause: ${err}`);
         }
     }
 

diff --git a/api/src/services/stardust/influx/influxDbService.ts b/api/src/services/stardust/influx/influxDbService.ts
@@ -121,6 +121,11 @@ export class InfluxDBService extends InfluxDbClient {
         return this._milestoneCache;
     }
 
+    public async fetchAnalyticsForMilestone(milestoneIndex: number) {
+        await this.collectMilestoneStatsByIndex(milestoneIndex);
+        return this._milestoneCache.get(milestoneIndex);
+    }
+
     public async fetchAnalyticsForMilestoneWithRetries(
         milestoneIndex: number
     ): Promise<IMilestoneAnalyticStats | undefined> {

diff --git a/api/src/services/stardust/influx/influxQueries.ts b/api/src/services/stardust/influx/influxQueries.ts
@@ -335,4 +335,15 @@ export const MILESTONE_STATS_QUERY = `
         last("no_payload_count") AS "noPayload"
     FROM "stardust_block_activity"
 `;
+export const MILESTONE_STATS_QUERY_BY_INDEX = `
+    SELECT
+        milestone_index AS "milestoneIndex",
+        tagged_data_count AS "taggedData",
+        milestone_count AS "milestone",
+        transaction_count AS "transaction",
+        treasury_transaction_count AS "treasuryTransaction",
+        no_payload_count AS "noPayload"
+    FROM "stardust_block_activity"
+    WHERE "milestone_index" = $milestoneIndex
+`;
 
diff --git a/client/src/helpers/hooks/useBlockFeed.ts b/client/src/helpers/hooks/useBlockFeed.ts
@@ -27,6 +27,7 @@ export function useBlockFeed(network: string): [
     const resetCounter = useRef<number>(0);
     const [milestones, setMilestones] = useState<IMilestoneFeedItem[]>([]);
     const [latestMilestonIndex, setLatestMilestoneIndex] = useState<number | null>(null);
+    const latestMilestoneIndexRef = useRef<number | null>(latestMilestonIndex);
 
     const fetchLatestCachedMilestones = useCallback(async () => {
         if (apiClient) {
@@ -60,6 +61,10 @@ export function useBlockFeed(network: string): [
         };
     }, [network, feedProbe]);
 
+    useEffect(() => {
+        latestMilestoneIndexRef.current = latestMilestonIndex;
+    }, [latestMilestonIndex]);
+
     useEffect(() => {
         // eslint-disable-next-line no-void
         void fetchLatestCachedMilestones();
@@ -69,7 +74,7 @@ export function useBlockFeed(network: string): [
             const onMilestoneUpdate = (newMilestone: IFeedMilestoneData) => {
                 lastUpdateTime.current = Date.now();
                 if (isMounted) {
-                    if (isMounted && (latestMilestonIndex ?? 0) < newMilestone.milestoneIndex) {
+                    if (isMounted && (latestMilestoneIndexRef.current ?? 0) < newMilestone.milestoneIndex) {
                         setLatestMilestoneIndex(newMilestone.milestoneIndex);
                     }
                     if (isMounted) {