'use strict';
const { fork } = require('child_process');
const { minifyHtml } = require('core/strings');
const { readFile } = require('fs/promises');
const { makeId } = require('core/makeid');
const buildDeps = {
rollup: require('rollup').rollup,
commonjs: require('@rollup/plugin-commonjs'),
css: require('rollup-plugin-css-only'),
resolve: require('@rollup/plugin-node-resolve').nodeResolve,
svelte: require('rollup-plugin-svelte'),
terser: require('rollup-plugin-terser').terser,
};
const icons = {
server: '',
outage: '',
external: '',
};
const servicesNotifiedAboutOutage = new Set();
let renderedDashboard = null;
async function processOutage({ outage, server, settings, onDateUpdated }) {
if (typeof onDateUpdated !== 'function') {
onDateUpdated = () => null;
}
for (const [ id, testResult ] of Object.entries(outage)) {
// Update check date
server.storage.store('smartyellow/webservice').update(
{ id },
{ $set: { lastChecked: new Date() } }
).then(() => onDateUpdated(id));
// Get service entry
const service = await server
.storage
.store('smartyellow/webservice')
.findOne({ id });
// Get last heartbeat
const heartbeat = await server
.storage
.store('smartyellow/webserviceheartbeat')
.find({ webservice: id })
.toArray();
const lastBeat = heartbeat[heartbeat.length - 1];
// Encountered an error while checking status
if (testResult.error) {
server.error('status: error while checking status of ' + id);
server.error(testResult);
}
// Service is down
else if (!testResult.serviceUp) {
// Don't perform automatic actions if already done
if ((lastBeat && lastBeat.down == false) || !lastBeat) {
// Insert heartbeat if last one is not valid anymore
try {
server.storage.store('smartyellow/webserviceheartbeat').insert({
id: makeId(10),
down: true,
webservice: id,
testResult,
date: new Date(),
});
}
catch (err) {
server.error('status: could not save web service heartbeat');
server.error(err);
}
}
}
// Service up
else {
// Don't perform automatic actions if already done
if ((lastBeat && lastBeat.down == true) || !lastBeat) {
// Insert heartbeat if last one is not valid anymore
try {
await server.storage.store('smartyellow/webserviceheartbeat').insert({
id: makeId(10),
down: false,
webservice: id,
testResult,
date: new Date(),
});
}
catch (err) {
server.warn('status: could not save web service heartbeat');
server.warn(err);
}
}
}
}
return;
}
module.exports = {
// Friendly name
name: 'Status',
// Brief description of this plugin
purpose: 'Keep track of the status of your web services automatically and get notified when a service is down.',
// Version of this plugin
version: '1.0.0',
// Name of the plugin author
author: 'Romein van Buren',
// Name of vendor of this plugin
vendor: 'Smart Yellow',
// Array of plugins this plugin depends on
requires: [ 'webdesq/storage' ],
// Features this plugin offers
features: {
seeServices: {
description: 'See all web services',
},
createServices: {
description: 'Create new web services',
requires: [ 'seeServices' ],
},
editServices: {
description: 'Edit web services',
requires: [ 'seeServices' ],
},
deleteServices: {
description: 'Delete web services',
requires: [ 'editServices' ],
},
seeOutages: {
description: 'See all outages',
requires: [ 'seeServices' ],
},
createOutages: {
description: 'Create new outages',
requires: [ 'seeServices', 'seeOutages' ],
},
editOutages: {
description: 'Edit outages',
requires: [ 'seeServices', 'seeOutages' ],
},
deleteOutages: {
description: 'Delete outages',
requires: [ 'seeServices', 'editOutages' ],
},
seeMonitor: {
description: 'See the monitor',
},
},
icon: icons.server,
entities: {
webservice: 'webservice.js',
webserviceoutage: 'webserviceoutage.js',
webserviceheartbeat: 'webserviceheartbeat.js',
},
settings: {
clusters: {
type: 'keys',
label: 'clusters',
description: 'Clusters can be used to catogorise web services into groups.',
default: {},
},
serviceTags: {
type: 'keys',
label: 'service tags',
description: 'Tags that can be assigned to web services to categorise them.',
default: {},
},
outageTags: {
type: 'keys',
label: 'outage tags',
description: 'Tags that can be assigned to outage messages to categorise them.',
default: {},
},
// emailSender: {
// type: 'string',
// label: 'notification sender',
// description: 'Sender of notifications about service statuses. Format: Name ',
// default: '',
// },
// emailRecipient: {
// type: 'array',
// label: 'notification recipients',
// description: 'Recipients of notifications about service statuses. Format: Name ',
// default: [],
// },
smsRecipients: {
type: 'array',
label: 'SMS recipients',
description: 'Recipients of SMSes about service statuses.',
default: [],
},
draftOutageEntries: {
type: 'boolean',
label: 'draft outage entries',
description: 'Automatically draft an outage entry when a service is down?',
default: true,
},
},
gui: {
components: [
'formautotestfield.svelte',
'formoutagetablefield.svelte',
],
modules: () => [
{ path: 'webservices.svelte',
requires: [ 'seeServices', 'editServices' ],
menu: {
cluster: 'web service status',
icon: icons.server,
title: 'web services',
},
},
{ path: 'webserviceoutages.svelte',
requires: [ 'seeServices', 'editServices' ],
menu: {
cluster: 'web service status',
icon: icons.outage,
title: 'outages',
},
},
{ path: 'webservicedashboard.svelte',
menu: {
cluster: 'web service status',
icon: icons.external,
title: 'dashboard',
},
},
],
widgets: () => [
{ path: 'webservicestatus.svelte',
title: 'Web service status',
purpose: 'Monitor web service status',
defaults: {
title: 'Web service status',
},
},
],
},
jobs: ({ server, settings }) => [
{ id: 'autotest',
purpose: 'Check whether services are up and send a notification if not.',
mandatory: false,
runAtBoot: true,
active: true,
interval: 60 * 1000,
action: async () => {
const services = await server
.storage
.store('smartyellow/webservice')
.find()
.toArray();
if (!services.length) {
return;
}
const runtime = fork(__dirname + '/runtime.js');
runtime.send({ command: 'testAll', services });
runtime.on('message', message => {
if (message.error) {
server.error('status: runtime error');
server.error(message.error);
}
else if (message.outage) {
processOutage({ outage: message.outage, server, settings });
}
});
},
},
],
hooks: ({ server, settings }) => [
{ id: 'startDashboardSocket',
event: 'boot',
order: 100,
purpose: 'Start the websocket for the dashboard after boot',
handler: () => {
let downIdsBefore = [];
let downIdsAfter = [];
let newOutage = false;
const mapService = (s, beat) => ({
id: s.id,
name: s.name,
cluster: s.cluster,
lastBeat: beat,
checked: s.lastChecked,
});
server.ws({
route: '/status/dashboard/socket',
onOpen: async ws => {
async function sendStatuses() {
const services = await server.storage
.store('smartyellow/webservice')
.find({ public: true })
.toArray();
const heartbeats = await server.storage
.store('smartyellow/webserviceheartbeat')
.find({ webservice: { $in: services.map(s => s.id) } })
.sort({ date: -1 })
.toArray();
const tiles = [];
for (let service of services) {
const beat = heartbeats.find(b => b.webservice === service.id);
service = mapService(service, beat);
const tile = {
service: service,
serviceId: service.id,
badges: [],
prio: -1,
};
if (!beat) {
tile.prio = -1; // no data (grey)
tile.statusText = 'no data';
}
else if (beat.down) {
tile.prio = 2; // down (red)
tile.statusText = 'down';
downIdsAfter.push(tile.serviceId);
}
else {
tile.prio = 0; // ok (green)
tile.statusText = 'ok';
}
tiles.push(tile);
}
// Let other plugins enrich dashboard tiles with custom badges and priorities.
await server.executePreHooks('populateDashboardTiles', { tiles });
await server.executePostHooks('populateDashboardTiles', { tiles });
await server.executePostHooks('pupulateDashboardTiles', { tiles }); // backwards compatibility
// Check if there are new outages and report them by ringing a bell on the dashboard.
newOutage = false;
for (const id of downIdsAfter) {
if (!downIdsBefore.includes(id)) {
newOutage = true;
}
}
downIdsBefore = [ ...downIdsAfter ];
downIdsAfter = [];
try {
const json = JSON.stringify({ newOutage, tiles });
ws.send(json);
}
catch {
return;
}
}
// Send statuses on open and every 5 seconds.
sendStatuses();
setInterval(sendStatuses, 5000);
},
onUpgrade: async () => ({ id: makeId(10) }),
onMessage: async () => { /* do nothing */ },
});
},
},
{ id: 'autotestOnSave',
order: 500,
event: 'saveEntity',
entity: [ 'smartyellow/webservice' ],
purpose: 'Check whether services are up and send a notification if not.',
handler: ({ item }) => {
const runtime = fork(__dirname + '/runtime.js');
runtime.send({ command: 'testOne', service: item });
runtime.on('message', message => {
if (message.error) {
server.error('status: runtime error');
server.error(message.error);
}
else if (message.outage) {
processOutage({
outage: {
[item.id]: message.outage,
},
server,
settings,
});
}
});
},
},
{ id: 'sendSmsOnOutage',
order: 10,
event: 'populateDashboardTiles',
purpose: 'Sends an SMS when a tile with priority of 2 or higher is rendered',
handler: async ({ tiles }) => {
if ((typeof server.sendSMS !== 'function') || (settings.smsRecipients?.length < 1)) {
// sendSMS not available, or no recipients.
return;
}
const servicesToNotifyAbout = new Set();
for (const tile of tiles) {
if (tile.prio < 2) {
// Tile is not of priority 2 or higher. Remove its ID from servicesNotifiedAboutOutage.
servicesNotifiedAboutOutage.delete(tile.serviceId);
}
else {
// Tile is of high priority.
if (servicesNotifiedAboutOutage.has(tile.serviceId)) {
// Already notified about: do nothing.
}
else {
// Not yet notified about: send SMSes.
servicesNotifiedAboutOutage.add(tile.serviceId);
servicesToNotifyAbout.add(tile.service?.name?.en || tile.serviceId);
}
}
}
if (servicesToNotifyAbout.size > 0) {
// There are new critical tiles; send notification.
const string = [ ...servicesToNotifyAbout ].join(', ');
server.debug('Sending status SMSes for the following services: ', string);
settings.smsRecipients.forEach(phoneNumber => server.sendSMS({
to: phoneNumber,
msg: 'The following service/s is/are experiencing outage: ' + string,
}).catch(err => {
server.error('Failed to send status SMS');
server.error(err);
}));
}
},
},
],
routes: ({ server, settings }) => [
// Get all services
{ route: '/status/webservices',
method: 'get',
requires: 'smartyellow/status/seeServices',
handler: async (req, res, user) => {
const services = server.storage({ user }).store('smartyellow/webservice').find();
const result = await (req.headers['format'] == 'object' ? services.toObject() : services.toArray());
if (req.headers['format'] == 'object') {
for (const service of Object.keys(result)) {
result[service].heartbeat = await server
.storage
.store('smartyellow/webserviceheartbeat')
.find({ service: service.id })
.toArray();
}
}
else {
for (const [ i, service ] of result.entries()) {
result[i].heartbeat = await server
.storage
.store('smartyellow/webserviceheartbeat')
.find({ webservice: service.id })
.toArray();
}
}
res.json(result);
},
},
// Get details for specific service
{ route: '/status/webservices/:id',
method: 'get',
requires: 'smartyellow/status/seeServices',
handler: async (req, res, user) => {
const doc = await server.storage({ user }).store('smartyellow/webservice').get(req.params[0]);
const result = await server.validateEntity({
entity: 'smartyellow/webservice',
id: req.params[0],
data: doc,
validateOnly: true,
user: user,
isNew: false,
});
res.json(result);
},
},
{ route: '/status/webservices/:id/testnow',
method: 'post',
requires: 'smartyellow/status/editServices',
handler: async (req, res, user) => {
const item = await server.storage({ user }).store('smartyellow/webservice').get(req.params[0]);
const runtime = fork(__dirname + '/runtime.js');
runtime.send({ command: 'testOne', service: item });
runtime.on('message', async message => {
res.json(message);
if (message.error) {
server.error('status: runtime error');
server.error(message.error);
}
else if (message.outage) {
await processOutage({
outage: { [item.id]: message.outage },
onDateUpdated: () => server.publish('cms', 'smartyellow/status/reload'),
server,
settings,
});
}
});
},
},
{ route: '/status/webservices/search',
method: 'post',
requires: 'smartyellow/status/seeServices',
handler: async (req, res, user) => {
const filters = await server.getFilters({
entity: 'smartyellow/webservice',
user: user,
});
const q = server.storage({ user }).prepareQuery(filters, req.body.query, req.body.languages || false);
const result = await server.storage({ user }).store('smartyellow/webservice').find(q).sort({ 'log.created.on': -1 }).toArray();
res.json(result);
},
},
// Get filters for services
{ route: '/status/webservices/filters',
method: 'get',
requires: 'smartyellow/status/seeServices',
handler: async (req, res, user) => {
res.json(await server.getFilters({
entity: 'smartyellow/webservice',
user: user,
}));
},
},
// Get formats for services
{ route: '/status/webservices/formats',
method: 'get',
requires: 'smartyellow/status/seeServices',
handler: async (req, res, user) => {
const formats = await server.getFormats({
entity: 'smartyellow/webservice',
user: user,
});
res.json(formats);
},
},
// Create new service
{ route: '/status/webservices',
method: 'post',
requires: 'smartyellow/status/createServices',
handler: async (req, res, user) => {
// Validate the posted data
const result = await server.validateEntity({
entity: 'smartyellow/webservice',
data: req.body,
storeIfValid: true,
validateOnly: req.headers['init'],
form: req.headers['form'] || 'default',
isNew: true,
user: user,
});
if (!result.errors) {
// broadcast message to all clients to notify users have been changed
server.publish('cms', 'smartyellow/status/reload');
}
return res.json(result);
},
},
// Update existing service
{ route: '/status/webservices/:id',
method: 'put',
requires: 'smartyellow/status/editServices',
handler: async (req, res, user) => {
// Validate the posted data
const result = await server.validateEntity({
entity: 'smartyellow/webservice',
id: req.params[0],
data: req.body,
storeIfValid: true,
validateOnly: req.headers['init'],
form: req.headers['form'] || 'default',
isNew: false,
user: user,
});
if (!result.errors) {
// broadcast message to all clients to notify users have been changed
server.publish('cms', 'smartyellow/status/reload');
}
return res.json(result);
},
},
// Delete specific service
{ route: '/status/webservices/:id',
method: 'delete',
requires: 'smartyellow/status/deleteServices',
handler: async (req, res, user) => {
const result = await server.storage({ user }).store('smartyellow/webservice').delete({ id: req.params[0] });
if (!result.errors) {
// broadcast message to all clients to notify users have been changed
server.publish('cms', 'smartyellow/status/reload');
}
res.json(result);
},
},
// Get all outages
{ route: '/status/outages',
method: 'get',
requires: 'smartyellow/status/seeOutages',
handler: async (req, res, user) => {
const outages = server.storage({ user }).store('smartyellow/webserviceoutage').find();
const result = await (req.headers['format'] == 'object' ? outages.toObject() : outages.toArray());
res.json(result);
},
},
// Get details for specific outage
{ route: '/status/outages/:id',
method: 'get',
requires: 'smartyellow/status/seeOutages',
handler: async (req, res, user) => {
const doc = await server.storage({ user }).store('smartyellow/webserviceoutage').get(req.params[0]);
const result = await server.validateEntity({
entity: 'smartyellow/webserviceoutage',
id: req.params[0],
data: doc,
validateOnly: true,
user: user,
isNew: false,
});
res.json(result);
},
},
{ route: '/status/outages/search',
method: 'post',
requires: 'smartyellow/status/seeOutages',
handler: async (req, res, user) => {
const filters = await server.getFilters({
entity: 'smartyellow/webserviceoutage',
user: user,
});
const q = server.storage({ user }).prepareQuery(filters, req.body.query, req.body.languages || false);
const result = await server.storage({ user }).store('smartyellow/webserviceoutage').find(q).sort({ 'log.created.on': -1 }).toArray();
res.json(result);
},
},
// Get filters for outages
{ route: '/status/outages/filters',
method: 'get',
requires: 'smartyellow/status/seeOutages',
handler: async (req, res, user) => {
res.json(await server.getFilters({
entity: 'smartyellow/webserviceoutage',
user: user,
}));
},
},
// Get formats for outages
{ route: '/status/outages/formats',
method: 'get',
requires: 'smartyellow/status/seeOutages',
handler: async (req, res, user) => {
const formats = await server.getFormats({
entity: 'smartyellow/webserviceoutage',
user: user,
});
res.json(formats);
},
},
// Create new service
{ route: '/status/outages',
method: 'post',
requires: 'smartyellow/status/createOutages',
handler: async (req, res, user) => {
// Validate the posted data
const result = await server.validateEntity({
entity: 'smartyellow/webserviceoutage',
data: req.body,
storeIfValid: true,
validateOnly: req.headers['init'],
form: req.headers['form'] || 'default',
isNew: true,
user: user,
});
if (!result.errors) {
// broadcast message to all clients to notify users have been changed
server.publish('cms', 'smartyellow/status/reload');
}
return res.json(result);
},
},
// Update existing service
{ route: '/status/outages/:id',
method: 'put',
requires: 'smartyellow/status/editOutages',
handler: async (req, res, user) => {
// Validate the posted data
const result = await server.validateEntity({
entity: 'smartyellow/webserviceoutage',
id: req.params[0],
data: req.body,
storeIfValid: true,
validateOnly: req.headers['init'],
form: req.headers['form'] || 'default',
isNew: false,
user: user,
});
if (!result.errors) {
// broadcast message to all clients to notify users have been changed
server.publish('cms', 'smartyellow/status/reload');
}
return res.json(result);
},
},
// Delete specific service
{ route: '/status/outages/:id',
method: 'delete',
requires: 'smartyellow/status/deleteOutages',
handler: async (req, res, user) => {
const result = await server.storage({ user }).store('smartyellow/webserviceoutage').delete({ id: req.params[0] });
if (!result.errors) {
// broadcast message to all clients to notify users have been changed
server.publish('cms', 'smartyellow/status/reload');
}
res.json(result);
},
},
{ route: '/status/dashboard',
method: 'get',
handler: async (req, res) => {
// const cacheValid = !!renderedDashboard;
const cacheValid = false;
if (!cacheValid) {
// Build dashboard
let cssOutput = '';
try {
const bundle = await buildDeps.rollup({
input: __dirname + '/gui/dashboard/index.js',
plugins: [
buildDeps.svelte({ compilerOptions: { dev: false, generate: 'dom' } }),
buildDeps.css({ output: style => cssOutput = style }),
buildDeps.resolve({ browser: true, dedupe: [ 'svelte' ] }),
buildDeps.commonjs(),
buildDeps.terser(),
],
});
const { output } = await bundle.generate({
sourcemap: false,
format: 'iife',
name: 'app',
file: 'public/build/bundle.js',
});
renderedDashboard = {
map: output[0].map ? output[0].map.toUrl() : '',
code: output[0].code,
css: cssOutput,
globalCss: await readFile(
__dirname + '/gui/dashboard/app.css'
),
};
}
catch (error) {
server.error('status: error while building dashboard: ', error);
}
}
const dashboardHtml = minifyHtml(`
Web service status dashboard
`);
res.send(dashboardHtml);
},
},
{ route: '/status/dashboard/sound',
method: 'get',
handler: (req, res) => {
res.headers['content-type'] = 'audio/x-wav';
res.sendFile(__dirname + '/gui/sounds/bell.wav');
},
},
],
};