Fork endpoint testing processes

Signed-off-by: Romein van Buren <romein@vburen.nl>

index.js

@@ -1,8 +1,7 @@
 'use strict';
 
-const { makeId } = require('core/makeid');
+const { fork } = require('child_process');
-const { testService } = require('./lib/testservice');
+const { processOutage } = require('./lib/processoutage');
-const { testServices } = require('./lib/testservices');
 
 const guiCluster = 'web service status';
 const icons = {
@@ -181,7 +180,29 @@ module.exports = {
       runAtBoot: true,
       active: true,
       interval: Number(settings.autotestInterval) * 60 * 1000,
-      action: () => testServices({ server, settings, makeId }),
+      action: async () => {
+        const services = await server
+          .storage
+          .store('smartyellow/webservice')
+          .find({ autotestEnabled: true })
+          .toArray();
+
+        if (!services.length) {
+          return;
+        }
+
+        const runtime = fork(__dirname + '/lib/runtime.js');
+        runtime.send({ command: 'testAll', services });
+
+        runtime.on('message', message => {
+          if (message.error) {
+            server.error(message.error);
+          }
+          else if (message.outage) {
+            processOutage({ outage: message.outage, server, settings });
+          }
+        });
+      },
     },
   ],
 
@@ -191,12 +212,24 @@ module.exports = {
       event: 'saveEntity',
       entity: [ 'smartyellow/webservice' ],
       purpose: 'Check whether services are up and send a notification if not.',
-      handler: ({ item }) => testService({
+      handler: ({ item }) => {
-        service: item,
+        const runtime = fork(__dirname + '/lib/runtime.js');
-        server,
+        runtime.send({ command: 'testOne', service: item });
-        settings,
+        runtime.on('message', message => {
-        makeId,
+          if (message.error) {
-      }),
+            server.error(message.error);
+          }
+          else if (message.outage) {
+            processOutage({
+              outage: {
+                [item.id]: message.outage,
+              },
+              server,
+              settings,
+            });
+          }
+        });
+      },
     },
   ],
 

lib/processoutage.js

@@ -1,46 +1,37 @@
 'use strict';
 
-const { testEndpoints } = require('./testendpoints');
+const { makeId } = require('core/makeid');
-const { roundDate } = require('./utils');
 
-async function testService({ service, server, settings, makeId }) {
+async function processOutage({ outage, server, settings }) {
-  if (!service.autotestEnabled) {
+  for (const [ id, testResult ] of Object.entries(outage)) {
-    return;
+    // Update check date
-  }
-
-  try {
-    // Autotest the service
-    const result = await testEndpoints(service.autotest);
-    const name = service.name.en;
-
-    // Insert check date
     await server.storage.store('smartyellow/webservice').update(
-      { id: service.id },
+      { id },
       { $set: { lastChecked: new Date() } }
     );
 
-    // Get all heartbeats plus the last one
+    // Get service entry
+    const service = await server
+      .storage
+      .store('smartyellow/webservice')
+      .findOne({ id });
+
+    // Get last heartbeat
     const heartbeat = await server
       .storage
       .store('smartyellow/webserviceheartbeat')
-      .find({ webservice: service.id })
+      .find({ webservice: id })
       .toArray();
     const lastBeat = heartbeat[heartbeat.length - 1];
 
-    // Get date
+    // Encountered an error while checking status
-    const date = roundDate(new Date());
+    if (testResult.error) {
-
+      server.error('Error while checking status of ' + id);
-    // Error
+      server.error(testResult);
-    if (result.error) {
-      server.error('Error while checking status: ' + name);
-      server.error(result);
     }
 
-    // Service down
+    // Service is down
-    else if (!result.serviceUp) {
+    else if (!testResult.serviceUp) {
-      server.warn('Service down: ' + name);
-      server.warn(result);
-
       // Don't perform automatic actions if already done
       if ((lastBeat && lastBeat.down == false) || !lastBeat) {
         // Insert heartbeat if last one is not valid anymore
@@ -48,9 +39,9 @@ async function testService({ service, server, settings, makeId }) {
           await server.storage.store('smartyellow/webserviceheartbeat').insert({
             id: makeId(6),
             down: true,
-            webservice: service.id,
+            webservice: id,
-            testResult: result,
+            testResult,
-            date: date,
+            date: new Date(),
           });
         }
         catch (err) {
@@ -64,10 +55,10 @@ async function testService({ service, server, settings, makeId }) {
             await server.sendEmail({
               sender: settings.emailSender,
               to: settings.emailRecipient,
-              subject: `[outage] ${name} is down`,
+              subject: `[outage] ${service.name} is down`,
               body: `<p>Dear recipient,</p>
                     <p>This is to inform you about web service outage.
-                        The service <em>${name}</em> does not meet the
+                        The service <em>${service.name}</em> does not meet the
                         requirements for being considered as 'working'.</p>
                     <p>Please always check this before taking action.</p>`,
             });
@@ -87,7 +78,7 @@ async function testService({ service, server, settings, makeId }) {
               .insert({
                 id: makeId(6),
                 name: {
-                  en: `[automatic] Outage for ${name}`,
+                  en: `[automatic] Outage for ${service.name}`,
                 },
                 state: 'concept',
                 resolved: false,
@@ -96,7 +87,7 @@ async function testService({ service, server, settings, makeId }) {
                 notes: [ {
                   date: new Date(),
                   userId: 'system',
-                  text: `Automatically created outage. Reason: ${JSON.stringify(result, null, 2)}`,
+                  text: `Automatically created outage. Reason: ${JSON.stringify(testResult, null, 2)}`,
                 } ],
               });
           }
@@ -110,17 +101,16 @@ async function testService({ service, server, settings, makeId }) {
 
     // Service up
     else {
-      server.info('Service up: ' + name);
+      // Don't perform automatic actions if already done
-
-      // Insert heartbeat if last one is not valid anymore
       if ((lastBeat && lastBeat.down == true) || !lastBeat) {
+        // Insert heartbeat if last one is not valid anymore
         try {
           await server.storage.store('smartyellow/webserviceheartbeat').insert({
             id: makeId(6),
             down: false,
-            webservice: service.id,
+            webservice: id,
-            date: date,
+            testResult,
-            testResult: result,
+            date: new Date(),
           });
         }
         catch (err) {
@@ -130,9 +120,6 @@ async function testService({ service, server, settings, makeId }) {
       }
     }
   }
-  catch (err) {
-    server.error(err);
-  }
 }
 
-module.exports = { testService };
+module.exports = { processOutage };

lib/runtime.js

@@ -0,0 +1,46 @@
+'use strict';
+
+const { testEndpoints } = require('./testendpoints');
+
+process.on('message', async message => {
+  switch (message.command) {
+    case 'testAll':
+      if (!message.services) {
+        process.send({ error: 'services is not defined' });
+      }
+      else {
+        const ids = [];
+        const promises = [];
+        for (const service of message.services) {
+          if (service.autotestEnabled) {
+            ids.push(service.id);
+            promises.push(testEndpoints(service.autotest));
+          }
+        }
+
+        const result = await Promise.all(promises);
+        const mapped = {};
+        for (const [ i, id ] of ids.entries()) {
+          mapped[id] = result[i];
+        }
+
+        process.send({ outage: mapped });
+      }
+      break;
+
+    case 'testOne':
+      if (!message.service) {
+        process.send({ error: 'service is not defined' });
+      }
+      else {
+        const { service } = message;
+        const result = await testEndpoints(service.autotest);
+        process.send({ outage: result });
+      }
+      break;
+
+    default:
+      console.error('Unknown command:', message.command);
+      break;
+  }
+});

lib/testservices.js

@@ -1,19 +0,0 @@
-'use strict';
-
-const { testService } = require('./testservice');
-
-async function testServices({ server, settings, makeId }) {
-  const services = await server
-    .storage
-    .store('smartyellow/webservice')
-    .find({ autotestEnabled: true })
-    .toArray();
-
-  services.forEach(async service => {
-    if (service.autotestEnabled) {
-      testService({ service, server, settings, makeId });
-    }
-  });
-}
-
-module.exports = { testServices };

lib/utils.js

@@ -1,11 +0,0 @@
-'use strict';
-
-// Round minutes up to 10 and remove seconds
-// e.g. 15:34:51 -> 15:30:00
-function roundDate(d) {
-  d.setMinutes(Math.round(d.getMinutes() / 10) * 10);
-  d.setSeconds(0, 0);
-  return d;
-}
-
-module.exports = { roundDate };