',
default: [],
},
smsRecipients: {
type: 'array',
label: 'SMS recipients',
description: 'Recipients of SMSes about service statuses.',
default: [],
},
draftOutageEntries: {
type: 'boolean',
label: 'draft outage entries',
description: 'Automatically draft an outage entry when a service is down?',
default: true,
},
},
gui: {
components: [
'formautotestfield.svelte',
'formoutagetablefield.svelte',
],
modules: () => [
{ path: 'webservices.svelte',
requires: [ 'seeServices', 'editServices' ],
menu: {
cluster: 'web service status',
icon: icons.server,
title: 'web services',
},
},
{ path: 'webserviceoutages.svelte',
requires: [ 'seeServices', 'editServices' ],
menu: {
cluster: 'web service status',
icon: icons.outage,
title: 'outages',
},
},
{ path: 'webservicedashboard.svelte',
menu: {
cluster: 'web service status',
icon: icons.external,
title: 'dashboard',
},
},
],
widgets: () => [
{ path: 'webservicestatus.svelte',
title: 'Web service status',
purpose: 'Monitor web service status',
defaults: {
title: 'Web service status',
},
},
],
},
jobs: ({ server, settings }) => [
{ id: 'autotest',
purpose: 'Check whether services are up and send a notification if not.',
mandatory: false,
runAtBoot: true,
active: true,
interval: 60 * 1000,
action: async () => {
const services = await server
.storage
.store('smartyellow/webservice')
.find()
.toArray();
if (!services.length) {
return;
}
const runtime = fork(__dirname + '/runtime.js');
runtime.send({ command: 'testAll', services });
runtime.on('message', message => {
if (message.error) {
server.error('status: runtime error');
server.error(message.error);
}
else if (message.outage) {
processOutage({ outage: message.outage, server, settings });
}
});
},
},
],
hooks: ({ server, settings }) => [
{ id: 'startDashboardSocket',
event: 'boot',
order: 100,
purpose: 'Start the websocket for the dashboard after boot',
handler: () => {
const timers = {};
let tiles = [];
let downIdsBefore = [];
let downIdsAfter = [];
let newOutage = false;
function mapService(s, beat) {
return {
id: s.id,
name: s.name,
cluster: s.cluster,
lastBeat: beat,
checked: s.lastChecked,
};
}
async function getTiles() {
const services = await server.storage
.store('smartyellow/webservice')
.find({ public: true })
.toArray();
const heartbeats = await server.storage
.store('smartyellow/webserviceheartbeat')
.find({ webservice: { $in: services.map(s => s.id) } })
.sort({ date: -1 })
.toArray();
tiles = [];
for (let service of services) {
const beat = heartbeats.find(b => b.webservice === service.id);
service = mapService(service, beat);
const tile = {
service: service,
serviceId: service.id,
badges: [],
prio: -1,
};
if (!beat) {
tile.prio = -1; // no data (grey)
tile.statusText = 'no data';
}
else if (beat.down) {
tile.prio = 2; // down (red)
tile.statusText = 'down';
downIdsAfter.push(tile.serviceId);
}
else {
tile.prio = 0; // ok (green)
tile.statusText = 'ok';
}
tiles.push(tile);
}
// Let other plugins enrich dashboard tiles with custom badges and priorities.
await server.executePreHooks('populateDashboardTiles', { tiles });
await server.executePostHooks('populateDashboardTiles', { tiles });
await server.executePostHooks('pupulateDashboardTiles', { tiles }); // backwards compatibility
// Check if there are new outages and report them by ringing a bell on the dashboard.
newOutage = false;
for (const id of downIdsAfter) {
if (!downIdsBefore.includes(id)) {
newOutage = true;
}
}
downIdsBefore = [ ...downIdsAfter ];
downIdsAfter = [];
}
// Load tiles every 10 seconds.
setInterval(getTiles, 10000);
server.ws({
route: '/status/dashboard/socket',
onOpen: async ws => {
function sendTiles() {
try {
const json = JSON.stringify({ newOutage, tiles });
ws.send(json);
}
catch { /* noop */ }
}
// Send tiles on open and every 5 seconds.
sendTiles();
timers[ws.id] = setInterval(sendTiles, 5000);
},
onUpgrade: () => ({ id: makeId(10) }),
onMessage: () => { /* nevermind */ },
onClose: ws => {
clearInterval(timers[ws.id]);
delete timers[ws.id];
},
});
},
},
{ id: 'autotestOnSave',
order: 500,
event: 'saveEntity',
entity: [ 'smartyellow/webservice' ],
purpose: 'Check whether services are up and send a notification if not.',
handler: ({ item }) => {
const runtime = fork(__dirname + '/runtime.js');
runtime.send({ command: 'testOne', service: item });
runtime.on('message', message => {
if (message.error) {
server.error('status: runtime error');
server.error(message.error);
}
else if (message.outage) {
processOutage({
outage: {
[item.id]: message.outage,
},
server,
settings,
});
}
});
},
},
{ id: 'sendEmailOnOutage',
order: 10,
event: 'populateDashboardTiles',
purpose: 'Sends an e-mail when a tile with priority of 1 or higher is rendered',
handler: async ({ tiles }) => {
if ((typeof server.sendEmail !== 'function') || (typeof settings.emailSender !== 'string') || !settings.emailRecipients?.length) {
// Bad configuration or no email extension.
return;
}
const tilesToNotifyAbout = new Set();
for (const tile of tiles) {
if (tile.prio < 1) {
// Tile is not of priority 1 or higher. Remove its ID from servicesNotifiedAboutOutage.
servicesNotifiedAboutOutage.delete(tile.serviceId);
}
else {
// Tile is of sufficient priority.
if (servicesNotifiedAboutOutage.has(tile.serviceId)) {
// Already notified about: do nothing.
}
else {
// Not yet notified about: send email.
servicesNotifiedAboutOutage.add(tile.serviceId);
tilesToNotifyAbout.add(tile);
}
}
}
if (tilesToNotifyAbout.size > 0) {
server.debug('Sending status e-mails for the following services: ', [ ...tilesToNotifyAbout ].map(t => t.serviceId).join(', '));
const message = 'The following web services are updated to have priority 1 or higher:
';
settings.emailRecipients.forEach(address => server.sendEmail({
subject: `[alert] ${tilesToNotifyAbout.size} web service${tilesToNotifyAbout.size === 1 ? ' has' : 's have'} a high priority!`,
sender: settings.emailSender,
to: address,
body: message,
}).catch(err => {
server.error('status: failed to send e-mail');
server.error(err);
}));
}
},
},
{ id: 'sendSmsOnOutage',
order: 10,
event: 'populateDashboardTiles',
purpose: 'Sends an SMS when a tile with priority of 2 or higher is rendered',
handler: async ({ tiles }) => {
if ((typeof server.sendSMS !== 'function') || (settings.smsRecipients?.length < 1)) {
// sendSMS not available, or no recipients.
return;
}
const servicesToNotifyAbout = new Set();
for (const tile of tiles) {
if (tile.prio < 2) {
// Tile is not of priority 2 or higher. Remove its ID from servicesNotifiedAboutOutage.
servicesNotifiedAboutOutage.delete(tile.serviceId);
}
else {
// Tile is of high priority.
if (servicesNotifiedAboutOutage.has(tile.serviceId)) {
// Already notified about: do nothing.
}
else {
// Not yet notified about: send SMSes.
servicesNotifiedAboutOutage.add(tile.serviceId);
servicesToNotifyAbout.add(tile.service?.name?.en || tile.serviceId);
}
}
}
if (servicesToNotifyAbout.size > 0) {
// There are new critical tiles; send notification.
const string = [ ...servicesToNotifyAbout ].join(', ');
server.debug('Sending status SMSes for the following services: ', string);
settings.smsRecipients.forEach(phoneNumber => server.sendSMS({
to: phoneNumber,
msg: 'The following service/s is/are experiencing outage: ' + string,
}).catch(err => {
server.error('status: failed to send SMS');
server.error(err);
}));
}
},
},
],
routes: ({ server, settings }) => [
// Get all services
{ route: '/status/webservices',
method: 'get',
requires: 'smartyellow/status/seeServices',
handler: async (req, res, user) => {
const services = server.storage({ user }).store('smartyellow/webservice').find();
const result = await (req.headers['format'] == 'object' ? services.toObject() : services.toArray());
if (req.headers['format'] == 'object') {
for (const service of Object.keys(result)) {
result[service].heartbeat = await server
.storage
.store('smartyellow/webserviceheartbeat')
.find({ service: service.id })
.toArray();
}
}
else {
for (const [ i, service ] of result.entries()) {
result[i].heartbeat = await server
.storage
.store('smartyellow/webserviceheartbeat')
.find({ webservice: service.id })
.toArray();
}
}
res.json(result);
},
},
// Get details for specific service
{ route: '/status/webservices/:id',
method: 'get',
requires: 'smartyellow/status/seeServices',
handler: async (req, res, user) => {
const doc = await server.storage({ user }).store('smartyellow/webservice').get(req.params[0]);
const result = await server.validateEntity({
entity: 'smartyellow/webservice',
id: req.params[0],
data: doc,
validateOnly: true,
user: user,
isNew: false,
});
res.json(result);
},
},
{ route: '/status/webservices/:id/testnow',
method: 'post',
requires: 'smartyellow/status/editServices',
handler: async (req, res, user) => {
const item = await server.storage({ user }).store('smartyellow/webservice').get(req.params[0]);
const runtime = fork(__dirname + '/runtime.js');
runtime.send({ command: 'testOne', service: item });
runtime.on('message', async message => {
res.json(message);
if (message.error) {
server.error('status: runtime error');
server.error(message.error);
}
else if (message.outage) {
await processOutage({
outage: { [item.id]: message.outage },
onDateUpdated: () => server.publish('cms', 'smartyellow/status/reload'),
server,
settings,
});
}
});
},
},
{ route: '/status/webservices/search',
method: 'post',
requires: 'smartyellow/status/seeServices',
handler: async (req, res, user) => {
const filters = await server.getFilters({
entity: 'smartyellow/webservice',
user: user,
});
const q = server.storage({ user }).prepareQuery(filters, req.body.query, req.body.languages || false);
const result = await server.storage({ user }).store('smartyellow/webservice').find(q).sort({ 'log.created.on': -1 }).toArray();
res.json(result);
},
},
// Get filters for services
{ route: '/status/webservices/filters',
method: 'get',
requires: 'smartyellow/status/seeServices',
handler: async (req, res, user) => {
res.json(await server.getFilters({
entity: 'smartyellow/webservice',
user: user,
}));
},
},
// Get formats for services
{ route: '/status/webservices/formats',
method: 'get',
requires: 'smartyellow/status/seeServices',
handler: async (req, res, user) => {
const formats = await server.getFormats({
entity: 'smartyellow/webservice',
user: user,
});
res.json(formats);
},
},
// Create new service
{ route: '/status/webservices',
method: 'post',
requires: 'smartyellow/status/createServices',
handler: async (req, res, user) => {
// Validate the posted data
const result = await server.validateEntity({
entity: 'smartyellow/webservice',
data: req.body,
storeIfValid: true,
validateOnly: req.headers['init'],
form: req.headers['form'] || 'default',
isNew: true,
user: user,
});
if (!result.errors) {
// broadcast message to all clients to notify users have been changed
server.publish('cms', 'smartyellow/status/reload');
}
return res.json(result);
},
},
// Update existing service
{ route: '/status/webservices/:id',
method: 'put',
requires: 'smartyellow/status/editServices',
handler: async (req, res, user) => {
// Validate the posted data
const result = await server.validateEntity({
entity: 'smartyellow/webservice',
id: req.params[0],
data: req.body,
storeIfValid: true,
validateOnly: req.headers['init'],
form: req.headers['form'] || 'default',
isNew: false,
user: user,
});
if (!result.errors) {
// broadcast message to all clients to notify users have been changed
server.publish('cms', 'smartyellow/status/reload');
}
return res.json(result);
},
},
// Delete specific service
{ route: '/status/webservices/:id',
method: 'delete',
requires: 'smartyellow/status/deleteServices',
handler: async (req, res, user) => {
const result = await server.storage({ user }).store('smartyellow/webservice').delete({ id: req.params[0] });
if (!result.errors) {
// broadcast message to all clients to notify users have been changed
server.publish('cms', 'smartyellow/status/reload');
}
res.json(result);
},
},
// Get all outages
{ route: '/status/outages',
method: 'get',
requires: 'smartyellow/status/seeOutages',
handler: async (req, res, user) => {
const outages = server.storage({ user }).store('smartyellow/webserviceoutage').find();
const result = await (req.headers['format'] == 'object' ? outages.toObject() : outages.toArray());
res.json(result);
},
},
// Get details for specific outage
{ route: '/status/outages/:id',
method: 'get',
requires: 'smartyellow/status/seeOutages',
handler: async (req, res, user) => {
const doc = await server.storage({ user }).store('smartyellow/webserviceoutage').get(req.params[0]);
const result = await server.validateEntity({
entity: 'smartyellow/webserviceoutage',
id: req.params[0],
data: doc,
validateOnly: true,
user: user,
isNew: false,
});
res.json(result);
},
},
{ route: '/status/outages/search',
method: 'post',
requires: 'smartyellow/status/seeOutages',
handler: async (req, res, user) => {
const filters = await server.getFilters({
entity: 'smartyellow/webserviceoutage',
user: user,
});
const q = server.storage({ user }).prepareQuery(filters, req.body.query, req.body.languages || false);
const result = await server.storage({ user }).store('smartyellow/webserviceoutage').find(q).sort({ 'log.created.on': -1 }).toArray();
res.json(result);
},
},
// Get filters for outages
{ route: '/status/outages/filters',
method: 'get',
requires: 'smartyellow/status/seeOutages',
handler: async (req, res, user) => {
res.json(await server.getFilters({
entity: 'smartyellow/webserviceoutage',
user: user,
}));
},
},
// Get formats for outages
{ route: '/status/outages/formats',
method: 'get',
requires: 'smartyellow/status/seeOutages',
handler: async (req, res, user) => {
const formats = await server.getFormats({
entity: 'smartyellow/webserviceoutage',
user: user,
});
res.json(formats);
},
},
// Create new service
{ route: '/status/outages',
method: 'post',
requires: 'smartyellow/status/createOutages',
handler: async (req, res, user) => {
// Validate the posted data
const result = await server.validateEntity({
entity: 'smartyellow/webserviceoutage',
data: req.body,
storeIfValid: true,
validateOnly: req.headers['init'],
form: req.headers['form'] || 'default',
isNew: true,
user: user,
});
if (!result.errors) {
// broadcast message to all clients to notify users have been changed
server.publish('cms', 'smartyellow/status/reload');
}
return res.json(result);
},
},
// Update existing service
{ route: '/status/outages/:id',
method: 'put',
requires: 'smartyellow/status/editOutages',
handler: async (req, res, user) => {
// Validate the posted data
const result = await server.validateEntity({
entity: 'smartyellow/webserviceoutage',
id: req.params[0],
data: req.body,
storeIfValid: true,
validateOnly: req.headers['init'],
form: req.headers['form'] || 'default',
isNew: false,
user: user,
});
if (!result.errors) {
// broadcast message to all clients to notify users have been changed
server.publish('cms', 'smartyellow/status/reload');
}
return res.json(result);
},
},
// Delete specific service
{ route: '/status/outages/:id',
method: 'delete',
requires: 'smartyellow/status/deleteOutages',
handler: async (req, res, user) => {
const result = await server.storage({ user }).store('smartyellow/webserviceoutage').delete({ id: req.params[0] });
if (!result.errors) {
// broadcast message to all clients to notify users have been changed
server.publish('cms', 'smartyellow/status/reload');
}
res.json(result);
},
},
{ route: '/status/dashboard',
method: 'get',
handler: async (req, res) => {
// const cacheValid = !!renderedDashboard;
const cacheValid = false;
if (!cacheValid) {
// Build dashboard
let cssOutput = '';
try {
const bundle = await buildDeps.rollup({
input: __dirname + '/gui/dashboard/index.js',
plugins: [
buildDeps.svelte({ compilerOptions: { dev: false, generate: 'dom' } }),
buildDeps.css({ output: style => cssOutput = style }),
buildDeps.resolve({ browser: true, dedupe: [ 'svelte' ] }),
buildDeps.commonjs(),
buildDeps.terser(),
],
});
const { output } = await bundle.generate({
sourcemap: false,
format: 'iife',
name: 'app',
file: 'public/build/bundle.js',
});
renderedDashboard = {
map: output[0].map ? output[0].map.toUrl() : '',
code: output[0].code,
css: cssOutput,
globalCss: await readFile(
__dirname + '/gui/dashboard/app.css'
),
};
}
catch (error) {
server.error('status: error while building dashboard: ', error);
}
}
const dashboardHtml = minifyHtml(`
Web service status dashboard
`);
res.send(dashboardHtml);
},
},
{ route: '/status/dashboard/sound',
method: 'get',
handler: (req, res) => {
res.headers['content-type'] = 'audio/x-wav';
res.sendFile(__dirname + '/gui/sounds/bell.wav');
},
},
],
};