Feat: Add stat_hourly & min. max. ping (#4267)

Co-authored-by: Frank Elsinga <frank@elsinga.de>

db/knex_migrations/2023-12-21-0000-stat-ping-min-max.js

@@ -0,0 +1,24 @@
+exports.up = function (knex) {
+    return knex.schema
+        .alterTable("stat_daily", function (table) {
+            table.float("ping_min").notNullable().defaultTo(0).comment("Minimum ping during this period in milliseconds");
+            table.float("ping_max").notNullable().defaultTo(0).comment("Maximum ping during this period in milliseconds");
+        })
+        .alterTable("stat_minutely", function (table) {
+            table.float("ping_min").notNullable().defaultTo(0).comment("Minimum ping during this period in milliseconds");
+            table.float("ping_max").notNullable().defaultTo(0).comment("Maximum ping during this period in milliseconds");
+        });
+
+};
+
+exports.down = function (knex) {
+    return knex.schema
+        .alterTable("stat_daily", function (table) {
+            table.dropColumn("ping_min");
+            table.dropColumn("ping_max");
+        })
+        .alterTable("stat_minutely", function (table) {
+            table.dropColumn("ping_min");
+            table.dropColumn("ping_max");
+        });
+};

db/knex_migrations/2023-12-22-0000-hourly-uptime.js

@@ -0,0 +1,26 @@
+exports.up = function (knex) {
+    return knex.schema
+        .createTable("stat_hourly", function (table) {
+            table.increments("id");
+            table.comment("This table contains the hourly aggregate statistics for each monitor");
+            table.integer("monitor_id").unsigned().notNullable()
+                .references("id").inTable("monitor")
+                .onDelete("CASCADE")
+                .onUpdate("CASCADE");
+            table.integer("timestamp")
+                .notNullable()
+                .comment("Unix timestamp rounded down to the nearest hour");
+            table.float("ping").notNullable().comment("Average ping in milliseconds");
+            table.float("ping_min").notNullable().defaultTo(0).comment("Minimum ping during this period in milliseconds");
+            table.float("ping_max").notNullable().defaultTo(0).comment("Maximum ping during this period in milliseconds");
+            table.smallint("up").notNullable();
+            table.smallint("down").notNullable();
+
+            table.unique([ "monitor_id", "timestamp" ]);
+        });
+};
+
+exports.down = function (knex) {
+    return knex.schema
+        .dropTable("stat_hourly");
+};

server/uptime-calculator.js

@@ -34,16 +34,25 @@ class UptimeCalculator {
      */
     minutelyUptimeDataList = new LimitQueue(24 * 60);
 
+    /**
+     * Recent 30-day uptime, each item is a 1-hour interval
+     * Key: {number} DivisionKey
+     * @type {LimitQueue<number,string>}
+     */
+    hourlyUptimeDataList = new LimitQueue(30 * 24);
+
     /**
      * Daily uptime data,
      * Key: {number} DailyKey
      */
     dailyUptimeDataList = new LimitQueue(365);
 
-    lastDailyUptimeData = null;
     lastUptimeData = null;
+    lastHourlyUptimeData = null;
+    lastDailyUptimeData = null;
 
     lastDailyStatBean = null;
+    lastHourlyStatBean = null;
     lastMinutelyStatBean = null;
 
     /**
@@ -53,6 +62,10 @@ class UptimeCalculator {
      * @returns {Promise<UptimeCalculator>} UptimeCalculator
      */
     static async getUptimeCalculator(monitorID) {
+        if (!monitorID) {
+            throw new Error("Monitor ID is required");
+        }
+
         if (!UptimeCalculator.list[monitorID]) {
             UptimeCalculator.list[monitorID] = new UptimeCalculator();
             await UptimeCalculator.list[monitorID].init(monitorID);
@@ -108,13 +121,32 @@ class UptimeCalculator {
                 up: bean.up,
                 down: bean.down,
                 avgPing: bean.ping,
+                minPing: bean.pingMin,
+                maxPing: bean.pingMax,
+            });
+        }
+
+        // Load hourly data from database (recent 30 days only)
+        let hourlyStatBeans = await R.find("stat_hourly", " monitor_id = ? AND timestamp > ? ORDER BY timestamp", [
+            monitorID,
+            this.getHourlyKey(now.subtract(30, "day")),
+        ]);
+
+        for (let bean of hourlyStatBeans) {
+            let key = bean.timestamp;
+            this.hourlyUptimeDataList.push(key, {
+                up: bean.up,
+                down: bean.down,
+                avgPing: bean.ping,
+                minPing: bean.pingMin,
+                maxPing: bean.pingMax,
             });
         }
 
         // Load daily data from database (recent 365 days only)
         let dailyStatBeans = await R.find("stat_daily", " monitor_id = ? AND timestamp > ? ORDER BY timestamp", [
             monitorID,
-            this.getDailyKey(now.subtract(365, "day").unix()),
+            this.getDailyKey(now.subtract(365, "day")),
         ]);
 
         for (let bean of dailyStatBeans) {
@@ -123,6 +155,8 @@ class UptimeCalculator {
                 up: bean.up,
                 down: bean.down,
                 avgPing: bean.ping,
+                minPing: bean.pingMin,
+                maxPing: bean.pingMax,
             });
         }
     }
@@ -148,13 +182,16 @@ class UptimeCalculator {
         }
 
         let divisionKey = this.getMinutelyKey(date);
-        let dailyKey = this.getDailyKey(divisionKey);
+        let hourlyKey = this.getHourlyKey(date);
+        let dailyKey = this.getDailyKey(date);
 
         let minutelyData = this.minutelyUptimeDataList[divisionKey];
+        let hourlyData = this.hourlyUptimeDataList[hourlyKey];
         let dailyData = this.dailyUptimeDataList[dailyKey];
 
         if (flatStatus === UP) {
             minutelyData.up += 1;
+            hourlyData.up += 1;
             dailyData.up += 1;
 
             // Only UP status can update the ping
@@ -163,32 +200,57 @@ class UptimeCalculator {
                 // The first beat of the minute, the ping is the current ping
                 if (minutelyData.up === 1) {
                     minutelyData.avgPing = ping;
+                    minutelyData.minPing = ping;
+                    minutelyData.maxPing = ping;
                 } else {
                     minutelyData.avgPing = (minutelyData.avgPing * (minutelyData.up - 1) + ping) / minutelyData.up;
+                    minutelyData.minPing = Math.min(minutelyData.minPing, ping);
+                    minutelyData.maxPing = Math.max(minutelyData.maxPing, ping);
+                }
+
+                // Add avg ping
+                // The first beat of the hour, the ping is the current ping
+                if (hourlyData.up === 1) {
+                    hourlyData.avgPing = ping;
+                    hourlyData.minPing = ping;
+                    hourlyData.maxPing = ping;
+                } else {
+                    hourlyData.avgPing = (hourlyData.avgPing * (hourlyData.up - 1) + ping) / hourlyData.up;
+                    hourlyData.minPing = Math.min(hourlyData.minPing, ping);
+                    hourlyData.maxPing = Math.max(hourlyData.maxPing, ping);
                 }
 
                 // Add avg ping (daily)
                 // The first beat of the day, the ping is the current ping
-                if (minutelyData.up === 1) {
+                if (dailyData.up === 1) {
                     dailyData.avgPing = ping;
+                    dailyData.minPing = ping;
+                    dailyData.maxPing = ping;
                 } else {
                     dailyData.avgPing = (dailyData.avgPing * (dailyData.up - 1) + ping) / dailyData.up;
+                    dailyData.minPing = Math.min(dailyData.minPing, ping);
+                    dailyData.maxPing = Math.max(dailyData.maxPing, ping);
                 }
             }
 
         } else {
             minutelyData.down += 1;
+            hourlyData.down += 1;
             dailyData.down += 1;
         }
 
-        if (dailyData !== this.lastDailyUptimeData) {
-            this.lastDailyUptimeData = dailyData;
-        }
-
         if (minutelyData !== this.lastUptimeData) {
             this.lastUptimeData = minutelyData;
         }
 
+        if (hourlyData !== this.lastHourlyUptimeData) {
+            this.lastHourlyUptimeData = hourlyData;
+        }
+
+        if (dailyData !== this.lastDailyUptimeData) {
+            this.lastDailyUptimeData = dailyData;
+        }
+
         // Don't store data in test mode
         if (process.env.TEST_BACKEND) {
             log.debug("uptime-calc", "Skip storing data in test mode");
@@ -199,12 +261,24 @@ class UptimeCalculator {
         dailyStatBean.up = dailyData.up;
         dailyStatBean.down = dailyData.down;
         dailyStatBean.ping = dailyData.avgPing;
+        dailyStatBean.pingMin = dailyData.minPing;
+        dailyStatBean.pingMax = dailyData.maxPing;
         await R.store(dailyStatBean);
 
+        let hourlyStatBean = await this.getHourlyStatBean(hourlyKey);
+        hourlyStatBean.up = hourlyData.up;
+        hourlyStatBean.down = hourlyData.down;
+        hourlyStatBean.ping = hourlyData.avgPing;
+        hourlyStatBean.pingMin = hourlyData.minPing;
+        hourlyStatBean.pingMax = hourlyData.maxPing;
+        await R.store(hourlyStatBean);
+
         let minutelyStatBean = await this.getMinutelyStatBean(divisionKey);
         minutelyStatBean.up = minutelyData.up;
         minutelyStatBean.down = minutelyData.down;
         minutelyStatBean.ping = minutelyData.avgPing;
+        minutelyStatBean.pingMin = minutelyData.minPing;
+        minutelyStatBean.pingMax = minutelyData.maxPing;
         await R.store(minutelyStatBean);
 
         // Remove the old data
@@ -214,6 +288,11 @@ class UptimeCalculator {
             this.getMinutelyKey(date.subtract(24, "hour")),
         ]);
 
+        await R.exec("DELETE FROM stat_hourly WHERE monitor_id = ? AND timestamp < ?", [
+            this.monitorID,
+            this.getHourlyKey(date.subtract(30, "day")),
+        ]);
+
         return date;
     }
 
@@ -242,6 +321,31 @@ class UptimeCalculator {
         return this.lastDailyStatBean;
     }
 
+    /**
+     * Get the hourly stat bean
+     * @param {number} timestamp milliseconds
+     * @returns {Promise<import("redbean-node").Bean>} stat_hourly bean
+     */
+    async getHourlyStatBean(timestamp) {
+        if (this.lastHourlyStatBean && this.lastHourlyStatBean.timestamp === timestamp) {
+            return this.lastHourlyStatBean;
+        }
+
+        let bean = await R.findOne("stat_hourly", " monitor_id = ? AND timestamp = ?", [
+            this.monitorID,
+            timestamp,
+        ]);
+
+        if (!bean) {
+            bean = R.dispense("stat_hourly");
+            bean.monitor_id = this.monitorID;
+            bean.timestamp = timestamp;
+        }
+
+        this.lastHourlyStatBean = bean;
+        return this.lastHourlyStatBean;
+    }
+
     /**
      * Get the minutely stat bean
      * @param {number} timestamp milliseconds
@@ -268,11 +372,12 @@ class UptimeCalculator {
     }
 
     /**
+     * Convert timestamp to minutely key
      * @param {dayjs.Dayjs} date The heartbeat date
      * @returns {number} Timestamp
      */
     getMinutelyKey(date) {
-        // Convert the current date to the nearest minute (e.g. 2021-01-01 12:34:56 -> 2021-01-01 12:34:00)
+        // Truncate value to minutes (e.g. 2021-01-01 12:34:56 -> 2021-01-01 12:34:00)
         date = date.startOf("minute");
 
         // Convert to timestamp in second
@@ -283,6 +388,33 @@ class UptimeCalculator {
                 up: 0,
                 down: 0,
                 avgPing: 0,
+                minPing: 0,
+                maxPing: 0,
+            });
+        }
+
+        return divisionKey;
+    }
+
+    /**
+     * Convert timestamp to hourly key
+     * @param {dayjs.Dayjs} date The heartbeat date
+     * @returns {number} Timestamp
+     */
+    getHourlyKey(date) {
+        // Truncate value to hours (e.g. 2021-01-01 12:34:56 -> 2021-01-01 12:00:00)
+        date = date.startOf("hour");
+
+        // Convert to timestamp in second
+        let divisionKey = date.unix();
+
+        if (! (divisionKey in this.hourlyUptimeDataList)) {
+            this.hourlyUptimeDataList.push(divisionKey, {
+                up: 0,
+                down: 0,
+                avgPing: 0,
+                minPing: 0,
+                maxPing: 0,
             });
         }
 
@@ -291,13 +423,11 @@ class UptimeCalculator {
 
     /**
      * Convert timestamp to daily key
-     * @param {number} timestamp Timestamp
+     * @param {dayjs.Dayjs} date The heartbeat date
      * @returns {number} Timestamp
      */
-    getDailyKey(timestamp) {
+    getDailyKey(date) {
-        let date = dayjs.unix(timestamp);
+        // Truncate value to start of day (e.g. 2021-01-01 12:34:56 -> 2021-01-01 00:00:00)
-
-        // Convert the date to the nearest day (e.g. 2021-01-01 12:34:56 -> 2021-01-01 00:00:00)
         // Considering if the user keep changing could affect the calculation, so use UTC time to avoid this problem.
         date = date.utc().startOf("day");
         let dailyKey = date.unix();
@@ -307,12 +437,34 @@ class UptimeCalculator {
                 up: 0,
                 down: 0,
                 avgPing: 0,
+                minPing: 0,
+                maxPing: 0,
             });
         }
 
         return dailyKey;
     }
 
+    /**
+     * Convert timestamp to key
+     * @param {dayjs.Dayjs} datetime Datetime
+     * @param {"day" | "hour" | "minute"} type the type of data which is expected to be returned
+     * @returns {number} Timestamp
+     * @throws {Error} If the type is invalid
+     */
+    getKey(datetime, type) {
+        switch (type) {
+            case "day":
+                return this.getDailyKey(datetime);
+            case "hour":
+                return this.getHourlyKey(datetime);
+            case "minute":
+                return this.getMinutelyKey(datetime);
+            default:
+                throw new Error("Invalid type");
+        }
+    }
+
     /**
      * Flat status to UP or DOWN
      * @param {number} status the status which schould be turned into a flat status
@@ -333,21 +485,21 @@ class UptimeCalculator {
 
     /**
      * @param {number} num the number of data points which are expected to be returned
-     * @param {"day" | "minute"} type the type of data which is expected to be returned
+     * @param {"day" | "hour" | "minute"} type the type of data which is expected to be returned
      * @returns {UptimeDataResult} UptimeDataResult
      * @throws {Error} The maximum number of minutes greater than 1440
      */
     getData(num, type = "day") {
-        let key;
 
-        if (type === "day") {
+        if (type === "hour" && num > 24 * 30) {
-            key = this.getDailyKey(this.getCurrentDate().unix());
+            throw new Error("The maximum number of hours is 720");
-        } else {
-            if (num > 24 * 60) {
-                throw new Error("The maximum number of minutes is 1440");
-            }
-            key = this.getMinutelyKey(this.getCurrentDate());
         }
+        if (type === "minute" && num > 24 * 60) {
+            throw new Error("The maximum number of minutes is 1440");
+        }
+
+        // Get the current time period key based on the type
+        let key = this.getKey(this.getCurrentDate(), type);
 
         let total = {
             up: 0,
@@ -357,20 +509,37 @@ class UptimeCalculator {
         let totalPing = 0;
         let endTimestamp;
 
-        if (type === "day") {
+        // Get the eariest timestamp of the required period based on the type
-            endTimestamp = key - 86400 * (num - 1);
+        switch (type) {
-        } else {
+            case "day":
-            endTimestamp = key - 60 * (num - 1);
+                endTimestamp = key - 86400 * (num - 1);
+                break;
+            case "hour":
+                endTimestamp = key - 3600 * (num - 1);
+                break;
+            case "minute":
+                endTimestamp = key - 60 * (num - 1);
+                break;
+            default:
+                throw new Error("Invalid type");
         }
 
         // Sum up all data in the specified time range
         while (key >= endTimestamp) {
             let data;
 
-            if (type === "day") {
+            switch (type) {
-                data = this.dailyUptimeDataList[key];
+                case "day":
-            } else {
+                    data = this.dailyUptimeDataList[key];
-                data = this.minutelyUptimeDataList[key];
+                    break;
+                case "hour":
+                    data = this.hourlyUptimeDataList[key];
+                    break;
+                case "minute":
+                    data = this.minutelyUptimeDataList[key];
+                    break;
+                default:
+                    throw new Error("Invalid type");
             }
 
             if (data) {
@@ -379,27 +548,53 @@ class UptimeCalculator {
                 totalPing += data.avgPing * data.up;
             }
 
-            // Previous day
+            // Set key to the pervious time period
-            if (type === "day") {
+            switch (type) {
-                key -= 86400;
+                case "day":
-            } else {
+                    key -= 86400;
-                key -= 60;
+                    break;
+                case "hour":
+                    key -= 3600;
+                    break;
+                case "minute":
+                    key -= 60;
+                    break;
+                default:
+                    throw new Error("Invalid type");
             }
         }
 
         let uptimeData = new UptimeDataResult();
 
+        // If there is no data in the previous time ranges, use the last data?
         if (total.up === 0 && total.down === 0) {
-            if (type === "day" && this.lastDailyUptimeData) {
+            switch (type) {
-                total = this.lastDailyUptimeData;
+                case "day":
-                totalPing = total.avgPing * total.up;
+                    if (this.lastDailyUptimeData) {
-            } else if (type === "minute" && this.lastUptimeData) {
+                        total = this.lastDailyUptimeData;
-                total = this.lastUptimeData;
+                        totalPing = total.avgPing * total.up;
-                totalPing = total.avgPing * total.up;
+                    } else {
-            } else {
+                        return uptimeData;
-                uptimeData.uptime = 0;
+                    }
-                uptimeData.avgPing = null;
+                    break;
-                return uptimeData;
+                case "hour":
+                    if (this.lastHourlyUptimeData) {
+                        total = this.lastHourlyUptimeData;
+                        totalPing = total.avgPing * total.up;
+                    } else {
+                        return uptimeData;
+                    }
+                    break;
+                case "minute":
+                    if (this.lastUptimeData) {
+                        total = this.lastUptimeData;
+                        totalPing = total.avgPing * total.up;
+                    } else {
+                        return uptimeData;
+                    }
+                    break;
+                default:
+                    throw new Error("Invalid type");
             }
         }
 
@@ -416,6 +611,85 @@ class UptimeCalculator {
         return uptimeData;
     }
 
+    /**
+     * Get data in form of an array
+     * @param {number} num the number of data points which are expected to be returned
+     * @param {"day" | "hour" | "minute"} type the type of data which is expected to be returned
+     * @returns {Array<object>} uptime data
+     * @throws {Error} The maximum number of minutes greater than 1440
+     */
+    getDataArray(num, type = "day") {
+        if (type === "hour" && num > 24 * 30) {
+            throw new Error("The maximum number of hours is 720");
+        }
+        if (type === "minute" && num > 24 * 60) {
+            throw new Error("The maximum number of minutes is 1440");
+        }
+
+        // Get the current time period key based on the type
+        let key = this.getKey(this.getCurrentDate(), type);
+
+        let result = [];
+
+        let endTimestamp;
+
+        // Get the eariest timestamp of the required period based on the type
+        switch (type) {
+            case "day":
+                endTimestamp = key - 86400 * (num - 1);
+                break;
+            case "hour":
+                endTimestamp = key - 3600 * (num - 1);
+                break;
+            case "minute":
+                endTimestamp = key - 60 * (num - 1);
+                break;
+            default:
+                throw new Error("Invalid type");
+        }
+
+        // Get datapoints in the specified time range
+        while (key >= endTimestamp) {
+            let data;
+
+            switch (type) {
+                case "day":
+                    data = this.dailyUptimeDataList[key];
+                    break;
+                case "hour":
+                    data = this.hourlyUptimeDataList[key];
+                    break;
+                case "minute":
+                    data = this.minutelyUptimeDataList[key];
+                    break;
+                default:
+                    throw new Error("Invalid type");
+            }
+
+            if (data) {
+                data.timestamp = key;
+                result.push(data);
+            }
+
+            // Set key to the pervious time period
+            switch (type) {
+                case "day":
+                    key -= 86400;
+                    break;
+                case "hour":
+                    key -= 3600;
+                    break;
+                case "minute":
+                    key -= 60;
+                    break;
+                default:
+                    throw new Error("Invalid type");
+            }
+        }
+
+        return result;
+    }
+
     /**
      * Get the uptime data by duration
      * @param {'24h'|'30d'|'1y'} duration Only accept 24h, 30d, 1y
@@ -446,7 +720,7 @@ class UptimeCalculator {
      * @returns {UptimeDataResult} UptimeDataResult
      */
     get7Day() {
-        return this.getData(7);
+        return this.getData(168, "hour");
     }
 
     /**
@@ -464,7 +738,7 @@ class UptimeCalculator {
     }
 
     /**
-     * @returns {dayjs.Dayjs} Current date
+     * @returns {dayjs.Dayjs} Current datetime in UTC
      */
     getCurrentDate() {
         return dayjs.utc();
@@ -476,12 +750,12 @@ class UptimeDataResult {
     /**
      * @type {number} Uptime
      */
-    uptime;
+    uptime = 0;
 
     /**
      * @type {number} Average ping
      */
-    avgPing;
+    avgPing = null;
 }
 
 module.exports = {

test/backend-test/test-uptime-calculator.js

@@ -78,22 +78,27 @@ test("Test getMinutelyKey", async (t) => {
 
 test("Test getDailyKey", async (t) => {
     let c2 = new UptimeCalculator();
-    let dailyKey = c2.getDailyKey(dayjs.utc("2023-08-12 20:46:00").unix());
+    let dailyKey = c2.getDailyKey(dayjs.utc("2023-08-12 20:46:00"));
     assert.strictEqual(dailyKey, dayjs.utc("2023-08-12").unix());
 
     c2 = new UptimeCalculator();
-    dailyKey = c2.getDailyKey(dayjs.utc("2023-08-12 23:45:30").unix());
+    dailyKey = c2.getDailyKey(dayjs.utc("2023-08-12 23:45:30"));
     assert.strictEqual(dailyKey, dayjs.utc("2023-08-12").unix());
 
     // Edge case 1
     c2 = new UptimeCalculator();
-    dailyKey = c2.getDailyKey(dayjs.utc("2023-08-12 23:59:59").unix());
+    dailyKey = c2.getDailyKey(dayjs.utc("2023-08-12 23:59:59"));
     assert.strictEqual(dailyKey, dayjs.utc("2023-08-12").unix());
 
     // Edge case 2
     c2 = new UptimeCalculator();
-    dailyKey = c2.getDailyKey(dayjs.utc("2023-08-12 00:00:00").unix());
+    dailyKey = c2.getDailyKey(dayjs.utc("2023-08-12 00:00:00"));
     assert.strictEqual(dailyKey, dayjs.utc("2023-08-12").unix());
+
+    // Test timezone
+    c2 = new UptimeCalculator();
+    dailyKey = c2.getDailyKey(dayjs("Sat Dec 23 2023 05:38:39 GMT+0800 (Hong Kong Standard Time)"));
+    assert.strictEqual(dailyKey, dayjs.utc("2023-12-22").unix());
 });
 
 test("Test lastDailyUptimeData", async (t) => {