diff --git a/gui/dashboard/lib.js b/gui/dashboard/lib.js
index 12f1d3b..4cdc761 100644
--- a/gui/dashboard/lib.js
+++ b/gui/dashboard/lib.js
@@ -2,7 +2,7 @@ import { get, writable } from 'svelte/store';
import { quintOut } from 'svelte/easing';
import { crossfade } from 'svelte/transition';
-function createSettingsStore() {
+export const settings = (() => {
const defaults = {
theme: 'dark',
cols: 4,
@@ -14,10 +14,7 @@ function createSettingsStore() {
const s = writable(defaults);
function updateStorage(val) {
- window.localStorage.setItem('statusdash', JSON.stringify({
- ...defaults,
- ...val,
- }));
+ window.localStorage.setItem('statusdash', JSON.stringify({ ...defaults, ...val }));
s.set(val);
}
@@ -25,7 +22,7 @@ function createSettingsStore() {
let localStorage = {};
try {
- localStorage = JSON.parse(localStorageString);
+ localStorage = JSON.parse(localStorageString || '{}');
}
catch {
localStorage = {};
@@ -35,12 +32,10 @@ function createSettingsStore() {
return {
subscribe: s.subscribe,
- set: val => updateStorage(val),
+ set: val => updateStorage(val || {}),
update: val => updateStorage({ ...get(s), ...val }),
};
-}
-
-export const settings = createSettingsStore();
+})();
export const shuffle = crossfade({
fallback(node) {
diff --git a/index.js b/index.js
index 5fa3935..ad7db48 100644
--- a/index.js
+++ b/index.js
@@ -1,13 +1,18 @@
'use strict';
const { fork } = require('child_process');
-const { processOutage } = require('./lib/processoutage');
-const buildDashboard = require('./lib/dashboard/build');
-const createDashboardSocket = require('./lib/dashboard/socket');
const { minifyHtml } = require('core/strings');
const { readFile } = require('fs/promises');
+const { makeId } = require('core/makeid');
+const buildDeps = {
+ rollup: require('rollup').rollup,
+ commonjs: require('@rollup/plugin-commonjs'),
+ css: require('rollup-plugin-css-only'),
+ resolve: require('@rollup/plugin-node-resolve').nodeResolve,
+ svelte: require('rollup-plugin-svelte'),
+ terser: require('rollup-plugin-terser').terser,
+};
-const guiCluster = 'web service status';
const icons = {
server: '',
outage: '',
@@ -16,6 +21,142 @@ const icons = {
let renderedDashboard = null;
+async function processOutage({ outage, server, settings, onDateUpdated }) {
+ if (typeof onDateUpdated !== 'function') {
+ onDateUpdated = () => null;
+ }
+
+ for (const [ id, testResult ] of Object.entries(outage)) {
+ // Update check date
+ server.storage.store('smartyellow/webservice').update(
+ { id },
+ { $set: { lastChecked: new Date() } }
+ ).then(() => onDateUpdated(id));
+
+ // Get service entry
+ const service = await server
+ .storage
+ .store('smartyellow/webservice')
+ .findOne({ id });
+
+ // Get last heartbeat
+ const heartbeat = await server
+ .storage
+ .store('smartyellow/webserviceheartbeat')
+ .find({ webservice: id })
+ .toArray();
+ const lastBeat = heartbeat[heartbeat.length - 1];
+
+ // Encountered an error while checking status
+ if (testResult.error) {
+ server.error('status: error while checking status of ' + id);
+ server.error(testResult);
+ }
+
+ // Service is down
+ else if (!testResult.serviceUp) {
+ // Don't perform automatic actions if already done
+ if ((lastBeat && lastBeat.down == false) || !lastBeat) {
+ // Insert heartbeat if last one is not valid anymore
+ try {
+ server.storage.store('smartyellow/webserviceheartbeat').insert({
+ id: makeId(10),
+ down: true,
+ webservice: id,
+ testResult,
+ date: new Date(),
+ });
+ }
+ catch (err) {
+ server.error('status: could not save web service heartbeat');
+ server.error(err);
+ }
+
+ // Send e-mail notification
+ if (server.sendEmail && settings.emailSender && settings.emailRecipient) {
+ try {
+ const date = new Date().toLocaleString('en-GB', {
+ dateStyle: 'full',
+ timeStyle: 'full',
+ timeZone: 'Etc/UTC',
+ });
+
+ await server.sendEmail({
+ sender: settings.emailSender,
+ to: settings.emailRecipient,
+ subject: `[outage] ${service.name} is down`,
+ body: `Hello,
+
+As of ${date} UTC time, the service "${service.name}" does not meet the requirements for being
+considered as working.
+
+Technical information containing the reason for this alert:
+${JSON.stringify(testResult, null, 2)}
+
+Please always check this before taking action. This is an automated message.`,
+ });
+ }
+ catch (err) {
+ server.warn('status: could not send endpoint status notification e-mail');
+ server.warn(err);
+ }
+ }
+
+ // Draft outage entry
+ if (settings.draftOutageEntries) {
+ try {
+ server
+ .storage
+ .store('smartyellow/webserviceoutage')
+ .insert({
+ id: makeId(),
+ name: {
+ en: `[automatic] Outage for ${service.name.en}`,
+ },
+ state: 'concept',
+ resolved: false,
+ services: [ service.id ],
+ tags: [ 'automatically created' ],
+ notes: [ {
+ date: new Date(),
+ userId: 'system',
+ text: `Automatically created outage. Reason: ${JSON.stringify(testResult, null, 2)}`,
+ } ],
+ });
+ }
+ catch (err) {
+ server.warn('status: could not automatically draft outage entry');
+ server.warn(err);
+ }
+ }
+ }
+ }
+
+ // Service up
+ else {
+ // Don't perform automatic actions if already done
+ if ((lastBeat && lastBeat.down == true) || !lastBeat) {
+ // Insert heartbeat if last one is not valid anymore
+ try {
+ await server.storage.store('smartyellow/webserviceheartbeat').insert({
+ id: makeId(10),
+ down: false,
+ webservice: id,
+ testResult,
+ date: new Date(),
+ });
+ }
+ catch (err) {
+ server.warn('status: could not save web service heartbeat');
+ server.warn(err);
+ }
+ }
+ }
+ }
+
+ return;
+}
+
module.exports = {
// Friendly name
@@ -133,7 +274,7 @@ module.exports = {
{ path: 'webservices.svelte',
requires: [ 'seeServices', 'editServices' ],
menu: {
- cluster: guiCluster,
+ cluster: 'web service status',
icon: icons.server,
title: 'web services',
},
@@ -142,7 +283,7 @@ module.exports = {
{ path: 'webserviceoutages.svelte',
requires: [ 'seeServices', 'editServices' ],
menu: {
- cluster: guiCluster,
+ cluster: 'web service status',
icon: icons.outage,
title: 'outages',
},
@@ -150,7 +291,7 @@ module.exports = {
{ path: 'webservicedashboard.svelte',
menu: {
- cluster: guiCluster,
+ cluster: 'web service status',
icon: icons.external,
title: 'dashboard',
},
@@ -191,6 +332,7 @@ module.exports = {
runtime.on('message', message => {
if (message.error) {
+ server.error('status: runtime error');
server.error(message.error);
}
else if (message.outage) {
@@ -206,7 +348,108 @@ module.exports = {
event: 'boot',
order: 100,
purpose: 'Start the websocket for the dashboard after boot',
- handler: () => createDashboardSocket(server),
+ handler: () => {
+ const decoder = new TextDecoder('utf-8');
+ let downIdsBefore = [];
+ let downIdsAfter = [];
+
+ const mapService = (s, beat) => ({
+ id: s.id,
+ name: s.name,
+ cluster: s.cluster,
+ lastBeat: beat,
+ checked: s.lastChecked,
+ });
+
+ server.ws({
+ route: '/status/dashboard/socket',
+ onOpen: async ws => {
+ async function sendStatuses() {
+ const services = await server.storage
+ .store('smartyellow/webservice')
+ .find({ public: true })
+ .toArray();
+ const heartbeats = await server.storage
+ .store('smartyellow/webserviceheartbeat')
+ .find({ webservice: { $in: services.map(s => s.id) } })
+ .sort({ date: -1 })
+ .toArray();
+
+ const servicesUp = [];
+ const servicesDown = [];
+ const servicesUnknown = [];
+ downIdsAfter = [];
+
+ for (let service of services) {
+ const beat = heartbeats.find(b => b.webservice === service.id);
+ service = mapService(service, beat);
+
+ if (!beat) {
+ servicesUnknown.push(service);
+ }
+ else if (beat.down) {
+ servicesDown.push(service);
+ downIdsAfter.push(service.id);
+ }
+ else {
+ servicesUp.push(service);
+ }
+ }
+
+ const total = [
+ ...servicesUp,
+ ...servicesDown,
+ ...servicesUnknown,
+ ].length;
+
+ let newOutage = false;
+ for (const id of downIdsAfter) {
+ if (!downIdsBefore.includes(id)) {
+ newOutage = true;
+ }
+ }
+ downIdsBefore = JSON.parse(JSON.stringify(downIdsAfter));
+
+ try {
+ if (newOutage) {
+ ws.send(JSON.stringify({ cmd: 'bell' }));
+ }
+
+ ws.send(JSON.stringify({
+ cmd: 'data',
+ servicesUp,
+ servicesDown,
+ servicesUnknown,
+ total,
+ }));
+ }
+ catch {
+ return;
+ }
+ }
+
+ sendStatuses();
+ setInterval(sendStatuses, 5000);
+ },
+ onUpgrade: async () => ({ id: makeId(10) }),
+ onMessage: async (ws, msg) => {
+ msg = JSON.parse(decoder.decode(msg));
+
+ if (!msg || !msg.command) {
+ return;
+ }
+
+ switch (msg.command) {
+ case 'data':
+ ws.send('data');
+ return;
+
+ default:
+ return;
+ }
+ },
+ });
+ },
},
{ id: 'autotestOnSave',
@@ -219,6 +462,7 @@ module.exports = {
runtime.send({ command: 'testOne', service: item });
runtime.on('message', message => {
if (message.error) {
+ server.error('status: runtime error');
server.error(message.error);
}
else if (message.outage) {
@@ -296,6 +540,7 @@ module.exports = {
runtime.on('message', async message => {
res.json(message);
if (message.error) {
+ server.error('status: runtime error');
server.error(message.error);
}
else if (message.outage) {
@@ -558,14 +803,52 @@ module.exports = {
{ route: '/status/dashboard',
method: 'get',
handler: async (req, res) => {
- try {
- if (!renderedDashboard) {
- renderedDashboard = await buildDashboard();
- renderedDashboard.globalCss = await readFile(
- __dirname + '/gui/dashboard/app.css'
- );
+ if (!renderedDashboard) {
+ // Build dashboard
+ let cssOutput = '';
+
+ try {
+ const bundle = await buildDeps.rollup({
+ input: __dirname + '/gui/dashboard/index.js',
+ plugins: [
+ buildDeps.svelte({
+ compilerOptions: {
+ dev: false,
+ generate: 'dom',
+ },
+ }),
+ buildDeps.css({ output: style => cssOutput = style }),
+ buildDeps.resolve({
+ browser: true,
+ dedupe: [ 'svelte' ],
+ }),
+ buildDeps.commonjs(),
+ buildDeps.terser(),
+ ],
+ });
+
+ const { output } = await bundle.generate({
+ sourcemap: false,
+ format: 'iife',
+ name: 'app',
+ file: 'public/build/bundle.js',
+ });
+
+ renderedDashboard = {
+ map: output[0].map ? output[0].map.toUrl() : '',
+ code: output[0].code,
+ css: cssOutput,
+ globalCss: await readFile(
+ __dirname + '/gui/dashboard/app.css'
+ ),
+ };
}
- const dashboardHtml = minifyHtml(`
+ catch (error) {
+ server.error('status: error while building dashboard: ', error);
+ }
+ }
+
+ const dashboardHtml = minifyHtml(`
@@ -584,11 +867,7 @@ module.exports = {
`);
- res.send(dashboardHtml);
- }
- catch (error) {
- server.error('could not compile web service status dashboard', error);
- }
+ res.send(dashboardHtml);
},
},
diff --git a/lib/dashboard/build.js b/lib/dashboard/build.js
deleted file mode 100644
index d5cf555..0000000
--- a/lib/dashboard/build.js
+++ /dev/null
@@ -1,60 +0,0 @@
-'use strict';
-
-const { rollup } = require('rollup');
-const commonjs = require('@rollup/plugin-commonjs');
-const css = require('rollup-plugin-css-only');
-const { default: resolve } = require('@rollup/plugin-node-resolve');
-const svelte = require('rollup-plugin-svelte');
-const { terser } = require('rollup-plugin-terser');
-
-async function build() {
- let cssOutput = '';
-
- try {
- const bundle = await rollup({
- input: __dirname + '/../../gui/dashboard/index.js',
- plugins: [
- // Svelte
- svelte({
- compilerOptions: {
- dev: false,
- generate: 'dom',
- },
- }),
-
- // Extract CSS
- css({ output: style => cssOutput = style }),
-
- // Resolve dependencies
- resolve({
- browser: true,
- dedupe: [ 'svelte' ],
- }),
-
- // CommonJS functions
- commonjs(),
-
- // Minify
- terser(),
- ],
- });
-
- const { output } = await bundle.generate({
- sourcemap: false,
- format: 'iife',
- name: 'app',
- file: 'public/build/bundle.js',
- });
-
- return {
- map: output[0].map ? output[0].map.toUrl() : '',
- code: output[0].code,
- css: cssOutput,
- };
- }
- catch (error) {
- console.error('Error while building status dashboard: ', error);
- }
-}
-
-module.exports = build;
diff --git a/lib/dashboard/socket.js b/lib/dashboard/socket.js
deleted file mode 100644
index 6491f13..0000000
--- a/lib/dashboard/socket.js
+++ /dev/null
@@ -1,108 +0,0 @@
-'use strict';
-
-const { makeId } = require('core/makeid');
-
-const decoder = new TextDecoder('utf-8');
-let downIdsBefore = [];
-let downIdsAfter = [];
-
-const mapService = (s, beat) => ({
- id: s.id,
- name: s.name,
- cluster: s.cluster,
- lastBeat: beat,
- checked: s.lastChecked,
-});
-
-async function createDashboardSocket(server) {
- server.ws({
- route: '/status/dashboard/socket',
- onOpen: async ws => {
- async function sendStatuses() {
- const services = await server.storage
- .store('smartyellow/webservice')
- .find({ public: true })
- .toArray();
- const heartbeats = await server.storage
- .store('smartyellow/webserviceheartbeat')
- .find({ webservice: { $in: services.map(s => s.id) } })
- .sort({ date: -1 })
- .toArray();
-
- const servicesUp = [];
- const servicesDown = [];
- const servicesUnknown = [];
- downIdsAfter = [];
-
- for (let service of services) {
- const beat = heartbeats.find(b => b.webservice === service.id);
- service = mapService(service, beat);
-
- if (!beat) {
- servicesUnknown.push(service);
- }
- else if (beat.down) {
- servicesDown.push(service);
- downIdsAfter.push(service.id);
- }
- else {
- servicesUp.push(service);
- }
- }
-
- const total = [
- ...servicesUp,
- ...servicesDown,
- ...servicesUnknown,
- ].length;
-
- let newOutage = false;
- for (const id of downIdsAfter) {
- if (!downIdsBefore.includes(id)) {
- newOutage = true;
- }
- }
- downIdsBefore = JSON.parse(JSON.stringify(downIdsAfter));
-
- try {
- if (newOutage) {
- ws.send(JSON.stringify({ cmd: 'bell' }));
- }
-
- ws.send(JSON.stringify({
- cmd: 'data',
- servicesUp,
- servicesDown,
- servicesUnknown,
- total,
- }));
- }
- catch {
- return;
- }
- }
-
- sendStatuses();
- setInterval(sendStatuses, 5000);
- },
- onUpgrade: async () => ({ id: makeId(10) }),
- onMessage: async (ws, msg) => {
- msg = JSON.parse(decoder.decode(msg));
-
- if (!msg || !msg.command) {
- return;
- }
-
- switch (msg.command) {
- case 'data':
- ws.send('data');
- return;
-
- default:
- return;
- }
- },
- });
-}
-
-module.exports = createDashboardSocket;
diff --git a/lib/processoutage.js b/lib/processoutage.js
deleted file mode 100644
index b51df9f..0000000
--- a/lib/processoutage.js
+++ /dev/null
@@ -1,141 +0,0 @@
-'use strict';
-
-const { makeId } = require('core/makeid');
-
-async function processOutage({ outage, server, settings, onDateUpdated }) {
- if (typeof onDateUpdated !== 'function') {
- onDateUpdated = () => null;
- }
-
- for (const [ id, testResult ] of Object.entries(outage)) {
- // Update check date
- server.storage.store('smartyellow/webservice').update(
- { id },
- { $set: { lastChecked: new Date() } }
- ).then(() => onDateUpdated(id));
-
- // Get service entry
- const service = await server
- .storage
- .store('smartyellow/webservice')
- .findOne({ id });
-
- // Get last heartbeat
- const heartbeat = await server
- .storage
- .store('smartyellow/webserviceheartbeat')
- .find({ webservice: id })
- .toArray();
- const lastBeat = heartbeat[heartbeat.length - 1];
-
- // Encountered an error while checking status
- if (testResult.error) {
- server.error('Error while checking status of ' + id);
- server.error(testResult);
- }
-
- // Service is down
- else if (!testResult.serviceUp) {
- // Don't perform automatic actions if already done
- if ((lastBeat && lastBeat.down == false) || !lastBeat) {
- // Insert heartbeat if last one is not valid anymore
- try {
- server.storage.store('smartyellow/webserviceheartbeat').insert({
- id: makeId(10),
- down: true,
- webservice: id,
- testResult,
- date: new Date(),
- });
- }
- catch (err) {
- server.error('could not save web service heartbeat');
- server.error(err);
- }
-
- // Send e-mail notification
- if (server.sendEmail && settings.emailSender && settings.emailRecipient) {
- try {
- const date = new Date().toLocaleString('en-GB', {
- dateStyle: 'full',
- timeStyle: 'full',
- timeZone: 'Etc/UTC',
- });
- const a = await server.sendEmail({
- sender: settings.emailSender,
- to: settings.emailRecipient,
- subject: `[outage] ${service.name} is down`,
- body: `Dear recipient,
-
-As of ${date} UTC time, the service "${service.name}" does not meet the requirements for being
-considered as working.
-
-Technical information containing the reason for this alert:
-${JSON.stringify(testResult, null, 2)}
-
-Please always check this before taking action. This is an automated message.`,
- });
- console.log(a);
- }
- catch (err) {
- server.error('could not send endpoint status notification e-mail');
- server.error(err);
- }
- }
-
- // Draft outage entry
- if (settings.draftOutageEntries) {
- try {
- server
- .storage
- .store('smartyellow/webserviceoutage')
- .insert({
- id: makeId(),
- name: {
- en: `[automatic] Outage for ${service.name.en}`,
- },
- state: 'concept',
- resolved: false,
- services: [ service.id ],
- tags: [ 'automatically created' ],
- notes: [ {
- date: new Date(),
- userId: 'system',
- text: `Automatically created outage. Reason: ${JSON.stringify(testResult, null, 2)}`,
- } ],
- });
- }
- catch (err) {
- server.error('could not automatically draft outage entry');
- server.error(err);
- }
- }
- }
- }
-
- // Service up
- else {
- // Don't perform automatic actions if already done
- if ((lastBeat && lastBeat.down == true) || !lastBeat) {
- // Insert heartbeat if last one is not valid anymore
- try {
- await server.storage.store('smartyellow/webserviceheartbeat').insert({
- id: makeId(10),
- down: false,
- webservice: id,
- testResult,
- date: new Date(),
- });
- }
- catch (err) {
- server.error('could not save web service heartbeat');
- server.error(err);
- }
- }
- }
- }
-
- return;
-}
-
-module.exports = { processOutage };
diff --git a/lib/runtime.js b/lib/runtime.js
index 149b7ed..9a71368 100644
--- a/lib/runtime.js
+++ b/lib/runtime.js
@@ -1,6 +1,84 @@
'use strict';
-const { testEndpoints } = require('./testendpoints');
+const fetch = require('node-fetch');
+const { operators } = require('./operators');
+const { realValues } = require('./realvalues');
+const http = require('http');
+const https = require('https');
+
+// Force requests over IPv4
+const httpAgent = new http.Agent({ family: 4 });
+const httpsAgent = new https.Agent({ family: 4 });
+
+async function testEndpoints(endpoints) {
+ const output = {
+ serviceUp: undefined,
+ success: true,
+ error: false,
+ requirement: undefined,
+ realValue: undefined,
+ };
+
+ for (const endpoint of endpoints) {
+ try {
+ const headers = endpoint.headers.reduce((obj, item) => {
+ obj[item.name] = item.value;
+ return obj;
+ }, {});
+
+ const res = await fetch(endpoint.uri, {
+ headers,
+ agent: url => {
+ if (url.protocol === 'http:') {
+ return httpAgent;
+ }
+ return httpsAgent;
+ },
+ });
+ const body = await res.text();
+
+ endpoint.requirements.forEach(requirement => {
+ if (output.success === false || output.serviceUp === false) {
+ return;
+ }
+
+ if (!Object.keys(operators).includes(requirement.operator)) {
+ output.success = false;
+ output.error = 'unknown operator: ' + requirement.operator;
+ }
+
+ if (!Object.keys(realValues).includes(requirement.type)) {
+ output.success = false;
+ output.error = 'unknown type: ' + requirement.type;
+ }
+
+ const realValue = realValues[requirement.type]({ res, body });
+ let result = operators[requirement.operator](realValue, requirement.string);
+
+ if (!requirement.truth) {
+ result = !result;
+ }
+
+ if (!result) {
+ output.serviceUp = false;
+ output.requirement = requirement;
+ output.realValue = realValue;
+ }
+ else {
+ output.serviceUp = true;
+ }
+ });
+ }
+ catch (err) {
+ output.success = false;
+ output.serviceUp = false;
+ output.error = err;
+ console.error(err);
+ }
+ }
+
+ return output;
+}
process.on('message', async message => {
switch (message.command) {
diff --git a/lib/testendpoints.js b/lib/testendpoints.js
deleted file mode 100644
index 2b13f6e..0000000
--- a/lib/testendpoints.js
+++ /dev/null
@@ -1,83 +0,0 @@
-'use strict';
-
-const fetch = require('node-fetch');
-const { operators } = require('./operators');
-const { realValues } = require('./realvalues');
-const http = require('http');
-const https = require('https');
-
-// Force requests over IPv4
-const httpAgent = new http.Agent({ family: 4 });
-const httpsAgent = new https.Agent({ family: 4 });
-
-async function testEndpoints(endpoints) {
- const output = {
- serviceUp: undefined,
- success: true,
- error: false,
- requirement: undefined,
- realValue: undefined,
- };
-
- for (const endpoint of endpoints) {
- try {
- const headers = endpoint.headers.reduce((obj, item) => {
- obj[item.name] = item.value;
- return obj;
- }, {});
-
- const res = await fetch(endpoint.uri, {
- headers,
- agent: url => {
- if (url.protocol === 'http:') {
- return httpAgent;
- }
- return httpsAgent;
- },
- });
- const body = await res.text();
-
- endpoint.requirements.forEach(requirement => {
- if (output.success === false || output.serviceUp === false) {
- return;
- }
-
- if (!Object.keys(operators).includes(requirement.operator)) {
- output.success = false;
- output.error = 'unknown operator: ' + requirement.operator;
- }
-
- if (!Object.keys(realValues).includes(requirement.type)) {
- output.success = false;
- output.error = 'unknown type: ' + requirement.type;
- }
-
- const realValue = realValues[requirement.type]({ res, body });
- let result = operators[requirement.operator](realValue, requirement.string);
-
- if (!requirement.truth) {
- result = !result;
- }
-
- if (!result) {
- output.serviceUp = false;
- output.requirement = requirement;
- output.realValue = realValue;
- }
- else {
- output.serviceUp = true;
- }
- });
- }
- catch (err) {
- output.success = false;
- output.serviceUp = false;
- output.error = err;
- console.error(err);
- }
- }
-
- return output;
-}
-
-module.exports = { testEndpoints };