From c44f5a920921cb4593b809140a5269b48c2d593e Mon Sep 17 00:00:00 2001 From: Massimo Candela Date: Tue, 2 Mar 2021 04:42:14 +0100 Subject: [PATCH] added support for external config manager and for user groups definition file (#489) * refactoring to support external configuration module * allow for external connectors * refactored retrieval of roa flat data structure now delegated to rpk-validator lib * introduced support for external user group config file * increased test coverage * external group file documentation * improve documentation on generatePrefixListEveryDays (#482) * add comments on test connectors (#497) --- .github/workflows/main.yml | 6 +- .gitignore | 1 + .npmignore | 1 + config.yml.example | 9 ++ docs/configuration.md | 5 +- docs/usergroups.md | 31 ++++++ groups.yml.example | 12 +++ index.js | 6 +- package.json | 3 +- src/config/config.js | 124 +++++++++++++++++++++ src/config/configYml.js | 95 +++++++++++++++++ src/connectors/connectorFullThrottle.js | 25 +++-- src/connectors/connectorTest.js | 2 + src/env.js | 136 +----------------------- src/reports/report.js | 10 +- src/reports/reportAlerta.js | 34 +++--- src/reports/reportEmail.js | 64 ++++++----- src/reports/reportHTTP.js | 22 ++-- src/reports/reportSyslog.js | 1 - src/reports/reportWebex.js | 22 ++-- src/worker.js | 11 +- tests/1_config.js | 2 + tests/4_groups.js | 84 +++++++++++++++ tests/config.test.yml | 3 +- tests/groups.test.after.yml | 4 + tests/groups.test.yml | 4 + tests/npm_tests/testNpmLib.js | 66 ++++++++++++ 27 files changed, 550 insertions(+), 233 deletions(-) create mode 100644 groups.yml.example create mode 100644 src/config/config.js create mode 100644 src/config/configYml.js create mode 100644 tests/4_groups.js create mode 100644 tests/groups.test.after.yml create mode 100644 tests/groups.test.yml create mode 100644 tests/npm_tests/testNpmLib.js diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8fe28c2..d22f14e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -75,7 +75,7 @@ jobs: npm run test-core npm run test-generate npm run test-reports - + - name: Tests RPKI run: | rm -f -R .cache/ @@ -91,6 +91,10 @@ jobs: npm run test-proxy kill $ANYPROXY_PID + - name: Tests NPM + run: | + npm run test-npm + - name: Tests Kafka run: | sudo apt-get -y install tar diff --git a/.gitignore b/.gitignore index 4dcc0f1..89948c9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ config.yml prefixes.yml +groups.yml .idea/ node_modules/ bin/ diff --git a/.npmignore b/.npmignore index b918284..839bbc8 100644 --- a/.npmignore +++ b/.npmignore @@ -1,5 +1,6 @@ config.yml prefixes.yml +groups.yml .idea/ node_modules/ bin/ diff --git a/config.yml.example b/config.yml.example index c053526..0a977f5 100644 --- a/config.yml.example +++ b/config.yml.example @@ -275,6 +275,15 @@ generatePrefixListEveryDays: 0 monitoredPrefixesFiles: - prefixes.yml +############################ +# The file containing the user groups. +# User groups can be specified +# 1) directly above, in each report module; or +# 2) inside an external file, as specified below (disabled by default). +# Using an external file allows BGPalerter to auto-reload the user group definitions +# when the external file is changed. + +# groupsFile: groups.yml.example ############################ # HTTP proxy setting: diff --git a/docs/configuration.md b/docs/configuration.md index fa6959e..8de2e8f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -20,9 +20,9 @@ The following are common parameters which it is possible to specify in the confi |httpProxy| Defines the HTTP/HTTPS proxy server to be used by BGPalerter and its submodules (reporters/connectors/monitors). See [here](http-proxy.md) for more information. | A string | http://usr:psw@ prxy.org:8080 | No | |volume| Defines a directory that will contain the data that needs persistence. For example, configuration files and logs will be created in such directory (default to "./"). | A string | /home/bgpalerter/ | No | |persistStatus| If set to true, when BGPalerter is restarted the list of alerts already sent is recovered. This avoids duplicated alerts. The process must be able to write on disc inside `.cache/`. | A boolean | true | No | -|generatePrefixListEveryDays| This parameter allows to automatically re-generate the prefix list after the specified amount of days. Set to 0 to disable it. It works only if you have one prefix list file. | An integer | 0 | No | +|generatePrefixListEveryDays| This parameter allows to automatically re-generate the prefix list after the specified amount of days. Set to 0 to disable it. It works only if you have one prefix list file and if you have used BGPalerter to automatically generate the file (and not if you edited prefixes.yml manually). | An integer | 0 | No | |rpki| A dictionary containing the RPKI configuration (see [here](rpki.md) for more details). | | | Yes | - +|groupsFile| A file containing user groups definition (see [here](usergroups.md) for more details). | A string | groups.yml | No | The following are advanced parameters, please don't touch them if you are not doing research/experiments. @@ -38,6 +38,7 @@ The following are advanced parameters, please don't touch them if you are not do + ## Composition You can compose the tool with 3 main components: connectors, monitors, and reports. diff --git a/docs/usergroups.md b/docs/usergroups.md index 44a51de..1449bdb 100644 --- a/docs/usergroups.md +++ b/docs/usergroups.md @@ -11,6 +11,9 @@ By default, BGPalerter creates two user groups `noc` and `default` (since v1.27. You can create how many user groups you wish, for example to monitor resources of your customers and forward them the alerts about their resources without sending them administrative communications. +User groups can be specified directly in the report configuration on in an external yaml file. +Using an external file allows BGPalerter to auto-reload the user group definitions when the external file is changed. + ## Notify only specific users about specific prefixes Example of configuration. @@ -124,3 +127,31 @@ reports: default: _SLACK_WEBOOK_FOR_ADMIN_ group2: _SLACK_WEBOOK_FOR_GROUP2_ ``` + +## Define an external user groups file + +Edit `config.yml`, uncomment the `groupsFile` option, and add the position of the file (e.g., `groupsFile: groups.yml`). + +Create the user groups file as follows: + +```yaml +report_module_name1: + user_group_to_define: + list_of_contacts + +report_module_name2: + user_group_to_define: + list_of_contacts +``` +The format of the list of contacts depends on the report_module (e.g., emails for reportEmail, urls for reportHTTP). + +For example, for reportEmail: + +```yaml +reportEmail: + myGroup: + example@example.it +``` +In the repo there is a `config.yaml.example` file that you can use. + +It the file is changed, BGPalerter will auto-reload the user groups. diff --git a/groups.yml.example b/groups.yml.example new file mode 100644 index 0000000..7d5d3e4 --- /dev/null +++ b/groups.yml.example @@ -0,0 +1,12 @@ +# This file is an example of how you can define user groups + +# The structure is: +# report_module_name: +# user_group_to_define: +# list_of_contacts + +# The format of the list of contacts depends on the report_module (e.g., emails for reportEmail, urls for reportHTTP) + +reportEmail: + mygroup: + - example@example.it \ No newline at end of file diff --git a/index.js b/index.js index 0140ac7..93c54c0 100644 --- a/index.js +++ b/index.js @@ -187,5 +187,9 @@ switch(params._[0]) { global.DRY_RUN = !!params.t; if (global.DRY_RUN) console.log("Testing BGPalerter configuration. WARNING: remove -t option for production monitoring."); const Worker = require("./src/worker").default; - module.exports = new Worker(params.c, params.d); + module.exports = new Worker({ + configFile: params.c, + volume: params.d, + groupFile: params.E + }); } diff --git a/package.json b/package.json index 7a4576d..2701697 100644 --- a/package.json +++ b/package.json @@ -21,9 +21,10 @@ "test-proxy": "./node_modules/.bin/mocha --exit tests/proxy_tests/*.js --require @babel/register", "test-generate": "./node_modules/.bin/mocha --exit tests/generate_tests/*.js --require @babel/register", "test-kafka": "./node_modules/.bin/mocha --exit tests/kafka_tests/*.js --require @babel/register", + "test-npm": "./node_modules/.bin/mocha --exit tests/npm_tests/*.js --require @babel/register", "test-rpki": "./node_modules/.bin/mocha --exit tests/rpki_tests/tests.default.js --require @babel/register && ./node_modules/.bin/mocha --exit tests/rpki_tests/tests.external.js --require @babel/register && ./node_modules/.bin/mocha --exit tests/rpki_tests/tests.external-missing-roas.js --require @babel/register && rm -f -R .cache/ && ./node_modules/.bin/mocha --exit tests/rpki_tests/tests.external-roas.js --require @babel/register", "build": "./build.sh", - "compile": "rm -rf dist/ && ./node_modules/.bin/babel index.js -d dist && ./node_modules/.bin/babel src -d dist/src && cp package.json dist/package.json && cp README.md dist/README.md", + "compile": "rm -rf dist/ && ./node_modules/.bin/babel index.js -d dist && ./node_modules/.bin/babel src -d dist/src && cp package.json dist/package.json && cp README.md dist/README.md && cp .npm* dist/", "serve": "babel-node index.js", "inspect": "node --inspect --require @babel/register index.js", "update": "git update-index --assume-unchanged config.yml && git update-index --assume-unchanged prefixes.yml && git pull", diff --git a/src/config/config.js b/src/config/config.js new file mode 100644 index 0000000..cdcc7ac --- /dev/null +++ b/src/config/config.js @@ -0,0 +1,124 @@ +import axios from "axios"; + +export default class Config { + constructor(params) { + this.default = { + environment: "production", + connectors: [ + { + file: "connectorRIS", + name: "ris", + params: { + carefulSubscription: true, + url: "ws://ris-live.ripe.net/v1/ws/", + perMessageDeflate: true, + subscription: { + moreSpecific: true, + type: "UPDATE", + host: null, + socketOptions: { + includeRaw: false + } + } + } + } + ], + monitors: [ + { + file: "monitorHijack", + channel: "hijack", + name: "basic-hijack-detection", + params: { + thresholdMinPeers: 2 + } + }, + { + file: "monitorPath", + channel: "path", + name: "path-matching", + params: { + thresholdMinPeers: 1 + } + }, + { + file: "monitorNewPrefix", + channel: "newprefix", + name: "prefix-detection", + params: { + thresholdMinPeers: 3 + } + }, + { + file: "monitorVisibility", + channel: "visibility", + name: "withdrawal-detection", + params: { + thresholdMinPeers: 40 + } + }, + { + file: "monitorAS", + channel: "misconfiguration", + name: "as-monitor", + params: { + thresholdMinPeers: 3 + } + }, + { + file: "monitorRPKI", + channel: "rpki", + name: "rpki-monitor", + params: { + thresholdMinPeers: 1, + checkUncovered: false + } + } + ], + reports: [ + { + file: "reportFile", + channels: ["hijack", "newprefix", "visibility", "path", "misconfiguration", "rpki"] + } + ], + notificationIntervalSeconds: 86400, + alarmOnlyOnce: false, + monitoredPrefixesFiles: ["prefixes.yml"], + persistStatus: true, + generatePrefixListEveryDays: 0, + logging: { + directory: "logs", + logRotatePattern: "YYYY-MM-DD", + maxRetainedFiles: 10, + maxFileSizeMB: 15, + compressOnRotation: false, + }, + rpki: { + vrpProvider: "ntt", + preCacheROAs: true, + refreshVrpListMinutes: 15 + }, + checkForUpdatesAtBoot: true, + pidFile: "bgpalerter.pid", + fadeOffSeconds: 360, + checkFadeOffGroupsSeconds: 30 + }; + }; + + downloadDefault = () => { + return axios({ + url: 'https://raw.githubusercontent.com/nttgin/BGPalerter/master/config.yml.example', + method: 'GET', + responseType: 'blob', // important + }) + .then(response => response.data); + }; + + retrieve = () => { + throw new Error('The method retrieve must be implemented in the config connector'); + }; + + save = () => { + throw new Error('The method save must be implemented in the config connector'); + }; + +} \ No newline at end of file diff --git a/src/config/configYml.js b/src/config/configYml.js new file mode 100644 index 0000000..bc209f6 --- /dev/null +++ b/src/config/configYml.js @@ -0,0 +1,95 @@ +import Config from "./config"; +import yaml from "js-yaml"; +import fs from "fs"; +import path from "path"; + +export default class ConfigYml extends Config { + constructor(params) { + super(params); + this.configFile = global.EXTERNAL_CONFIG_FILE || + ((global.EXTERNAL_VOLUME_DIRECTORY) + ? global.EXTERNAL_VOLUME_DIRECTORY + 'config.yml' + : path.resolve(process.cwd(), 'config.yml')); + + this.groupsFile = global.EXTERNAL_GROUP_FILE; + + console.log("Loaded config:", this.configFile); + }; + + save = (config) => { + try { + fs.writeFileSync(this.configFile, yaml.dump(config)); + yaml.load(fs.readFileSync(this.configFile, 'utf8')); // Test readability and format + } catch (error) { + throw new Error("Cannot save the configuration in " + this.configFile); + } + }; + + retrieve = () => { + const ymlBasicConfig = yaml.dump(this.default); + + if (fs.existsSync(this.configFile)) { + try { + const config = yaml.load(fs.readFileSync(this.configFile, 'utf8')) || this.default; + this._readUserGroupsFiles(config); + + return config; + } catch (error) { + throw new Error("The file " + this.configFile + " is not valid yml: " + error.message.split(":")[0]); + } + } else { + console.log("Impossible to load config.yml. A default configuration file has been generated."); + + this.downloadDefault() + .then(data => { + fs.writeFileSync(this.configFile, data); + yaml.load(fs.readFileSync(this.configFile, 'utf8')); // Test readability and format + + this._readUserGroupsFiles(data); + }) + .catch(() => { + fs.writeFileSync(this.configFile, ymlBasicConfig); // Download failed, write simple default config + }); + + return this.default; + } + }; + + _readUserGroupsFiles = (config) => { + if (config.groupsFile) { + this.groupsFile = ((config.volume) + ? config.volume + config.groupsFile + : path.resolve(process.cwd(), config.groupsFile)); + + const userGroups = yaml.load(fs.readFileSync(this.groupsFile, 'utf8')); + + for (let report of config.reports) { + const name = report.file; + const groups = userGroups[name]; + if (userGroups[name]) { + report.params.userGroups = groups; + } + } + + fs.watchFile(this.groupsFile, () => { + if (this._watchPrefixFileTimer) { + clearTimeout(this._watchPrefixFileTimer) + } + this._watchPrefixFileTimer = setTimeout(() => { + const userGroups = yaml.load(fs.readFileSync(this.groupsFile, 'utf8')); + + for (let report of config.reports) { + const name = report.file; + const groups = userGroups[name]; + + if (userGroups[name]) { + report.params.userGroups = groups; + } + } + }, 5000); + }); + } + + }; + +} diff --git a/src/connectors/connectorFullThrottle.js b/src/connectors/connectorFullThrottle.js index f44110e..bc666fb 100644 --- a/src/connectors/connectorFullThrottle.js +++ b/src/connectors/connectorFullThrottle.js @@ -30,6 +30,9 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ + +// IMPORTANT: This Connector is just for stress tests during development. Please, ignore! + import Connector from "./connector"; import {AS, Path} from "../model"; @@ -119,17 +122,17 @@ export default class ConnectorFullThrottle extends Connector{ }); _startStream = () => { - setInterval(() => { - this.updates.forEach(message => this._message(message)); - this.updates.forEach(message => this._message(message)); - this.updates.forEach(message => this._message(message)); - this.updates.forEach(message => this._message(message)); - this.updates.forEach(message => this._message(message)); - this.updates.forEach(message => this._message(message)); - this.updates.forEach(message => this._message(message)); - this.updates.forEach(message => this._message(message)); - this.updates.forEach(message => this._message(message)); - this.updates.forEach(message => this._message(message)); + setInterval(() => { // just create a huge amount of useless messages + this.updates.forEach(this._message); + this.updates.forEach(this._message); + this.updates.forEach(this._message); + this.updates.forEach(this._message); + this.updates.forEach(this._message); + this.updates.forEach(this._message); + this.updates.forEach(this._message); + this.updates.forEach(this._message); + this.updates.forEach(this._message); + this.updates.forEach(this._message); }, 2); }; diff --git a/src/connectors/connectorTest.js b/src/connectors/connectorTest.js index dae39cd..c8f3255 100644 --- a/src/connectors/connectorTest.js +++ b/src/connectors/connectorTest.js @@ -30,6 +30,8 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +// IMPORTANT: This Connector is used by the automated tests. Please, ignore! + import Connector from "./connector"; import {AS, Path} from "../model"; import ipUtils from "ip-sub"; diff --git a/src/env.js b/src/env.js index 2a30574..6022b7f 100644 --- a/src/env.js +++ b/src/env.js @@ -30,151 +30,22 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import yaml from "js-yaml"; import fs from "fs"; -import path from "path"; import PubSub from './utils/pubSub'; import FileLogger from './utils/fileLogger'; import { version } from '../package.json'; import Storage from './utils/storages/storageFile'; -import axios from 'axios'; import url from 'url'; import RpkiUtils from './utils/rpkiUtils'; +import ConfigYml from './config/configYml'; +const configConnector = new (global.EXTERNAL_CONFIG_CONNECTOR || ConfigYml); const vector = { version: global.EXTERNAL_VERSION_FOR_TEST || version, - configFile: global.EXTERNAL_CONFIG_FILE || - ((global.EXTERNAL_VOLUME_DIRECTORY) - ? global.EXTERNAL_VOLUME_DIRECTORY + 'config.yml' - : path.resolve(process.cwd(), 'config.yml')), clientId: Buffer.from("bnR0LWJncGFsZXJ0ZXI=", 'base64').toString('ascii') }; -let config = { - environment: "production", - connectors: [ - { - file: "connectorRIS", - name: "ris", - params: { - carefulSubscription: true, - url: "ws://ris-live.ripe.net/v1/ws/", - perMessageDeflate: true, - subscription: { - moreSpecific: true, - type: "UPDATE", - host: null, - socketOptions: { - includeRaw: false - } - } - } - } - ], - monitors: [ - { - file: "monitorHijack", - channel: "hijack", - name: "basic-hijack-detection", - params: { - thresholdMinPeers: 2 - } - }, - { - file: "monitorPath", - channel: "path", - name: "path-matching", - params: { - thresholdMinPeers: 1 - } - }, - { - file: "monitorNewPrefix", - channel: "newprefix", - name: "prefix-detection", - params: { - thresholdMinPeers: 3 - } - }, - { - file: "monitorVisibility", - channel: "visibility", - name: "withdrawal-detection", - params: { - thresholdMinPeers: 40 - } - }, - { - file: "monitorAS", - channel: "misconfiguration", - name: "as-monitor", - params: { - thresholdMinPeers: 3 - } - }, - { - file: "monitorRPKI", - channel: "rpki", - name: "rpki-monitor", - params: { - thresholdMinPeers: 1, - checkUncovered: false - } - } - ], - reports: [ - { - file: "reportFile", - channels: ["hijack", "newprefix", "visibility", "path", "misconfiguration", "rpki"] - } - ], - notificationIntervalSeconds: 86400, - alarmOnlyOnce: false, - monitoredPrefixesFiles: ["prefixes.yml"], - persistStatus: true, - generatePrefixListEveryDays: 0, - logging: { - directory: "logs", - logRotatePattern: "YYYY-MM-DD", - maxRetainedFiles: 10, - maxFileSizeMB: 15, - compressOnRotation: false, - }, - rpki: { - vrpProvider: "ntt", - preCacheROAs: true, - refreshVrpListMinutes: 15 - }, - checkForUpdatesAtBoot: true, - pidFile: "bgpalerter.pid", - fadeOffSeconds: 360, - checkFadeOffGroupsSeconds: 30 -}; - -const ymlBasicConfig = yaml.dump(config); - -if (fs.existsSync(vector.configFile)) { - try { - config = yaml.load(fs.readFileSync(vector.configFile, 'utf8')) || config; - } catch (error) { - throw new Error("The file " + vector.configFile + " is not valid yml: " + error.message.split(":")[0]); - } -} else { - console.log("Impossible to load config.yml. A default configuration file has been generated."); - - axios({ - url: 'https://raw.githubusercontent.com/nttgin/BGPalerter/master/config.yml.example', - method: 'GET', - responseType: 'blob', // important - }) - .then((response) => { - fs.writeFileSync(vector.configFile, response.data); - yaml.load(fs.readFileSync(vector.configFile, 'utf8')); // Test readability and format - }) - .catch(() => { - fs.writeFileSync(vector.configFile, ymlBasicConfig); // Download failed, write simple default config - }) -} +const config = configConnector.retrieve(); if (global.DRY_RUN) { config.connectors = [{ @@ -268,6 +139,7 @@ config.reports = (config.reports || []) .map(item => { return { + file: item.file, class: require("./reports/" + item.file).default, channels: item.channels, params: item.params diff --git a/src/reports/report.js b/src/reports/report.js index 228f285..e741946 100644 --- a/src/reports/report.js +++ b/src/reports/report.js @@ -37,13 +37,13 @@ import axios from "axios"; import axiosEnrich from "../utils/axiosEnrich"; export default class Report { - constructor(channels, params, env) { this.config = env.config; this.logger = env.logger; this.pubSub = env.pubSub; this.params = params; + this.enabled = true; for (let channel of channels){ env.pubSub.subscribe(channel, (content, message) => { @@ -188,8 +188,12 @@ export default class Report { return template.replace(/\${([^}]*)}/g, (r,k)=>context[k]); }; - report = (message, content) => { throw new Error('The method report must be implemented'); - } + }; + + getUserGroup = (group) => { + throw new Error('The method getUserGroup must be implemented'); + }; + } diff --git a/src/reports/reportAlerta.js b/src/reports/reportAlerta.js index e640452..7f896ca 100644 --- a/src/reports/reportAlerta.js +++ b/src/reports/reportAlerta.js @@ -39,19 +39,12 @@ export default class ReportAlerta extends Report { this.environment = env.config.environment; this.enabled = true; - if (!this.params.urls || !Object.keys(this.params.urls).length) { + if (!this.getUserGroup("default")) { this.logger.log({ level: 'error', - message: "Alerta reporting is not enabled: no group is defined" + message: "Alerta reporting is not enabled: no default group defined" }); this.enabled = false; - } else { - if (!this.params.urls["default"]) { - this.logger.log({ - level: 'error', - message: "In urls, for reportAlerta, a group named 'default' is required for communications to the admin." - }); - } } this.headers = { @@ -75,14 +68,11 @@ export default class ReportAlerta extends Report { if (this.params.resource_templates) { this.logger.log({ level: 'info', - message: "The resource_templates parameter will be soon deprecated in favour of resourceTemplates. Please update your config.yml file accordingly." + message: "The resource_templates parameter is deprecated in favour of resourceTemplates. Please update your config.yml file accordingly." }); } - const resource = this.params.resourceTemplates[channel] || - this.params.resource_templates[channel] || - this.params.resourceTemplates["default"] || - this.params.resource_templates["default"]; + const resource = this.params.resourceTemplates[channel] || this.params.resourceTemplates["default"]; this.axios({ url: url + "/alert", @@ -107,18 +97,24 @@ export default class ReportAlerta extends Report { }) }; + getUserGroup = (group) => { + const groups = this.params.urls || this.params.userGroups; + + return groups[group] || groups["default"]; + }; + report = (channel, content) => { if (this.enabled){ let groups = content.data.map(i => i.matchedRule.group).filter(i => i != null); - groups = (groups.length) ? [...new Set(groups)] : Object.keys(this.params.urls); // If there is no specific group defined, send to all of them + groups = (groups.length) ? [...new Set(groups)] : [this.getUserGroup("default")]; for (let group of groups) { - if (this.params.urls[group]) { - this._createAlertaAlert(this.params.urls[group], channel, content); + const url = this.getUserGroup(group); + if (url) { + this._createAlertaAlert(url, channel, content); } } } - - } + }; } diff --git a/src/reports/reportEmail.js b/src/reports/reportEmail.js index 9ce5e39..0652532 100644 --- a/src/reports/reportEmail.js +++ b/src/reports/reportEmail.js @@ -36,28 +36,20 @@ import emailTemplates from "./email_templates/emailTemplates"; export default class ReportEmail extends Report { - constructor(channels,params, env) { + constructor(channels, params, env) { super(channels, params, env); this.emailTemplates = new emailTemplates(this.logger); - this.templates = {}; this.emailBacklog = []; - if (!this.params.notifiedEmails || !Object.keys(this.params.notifiedEmails).length) { + if (!this.getUserGroup("default")) { + this.enabled = false; this.logger.log({ level: 'error', - message: "Email reporting is not enabled: no group is defined" + message: "In notifiedEmails, for reportEmail, a group named 'default' is required for communications to the admin." }); - } else { - if (!this.params.notifiedEmails["default"] || !this.params.notifiedEmails["default"].length) { - this.logger.log({ - level: 'error', - message: "In notifiedEmails, for reportEmail, a group named 'default' is required for communications to the admin." - }); - } - this.transporter = nodemailer.createTransport(this.params.smtp); for (let channel of channels) { @@ -71,6 +63,14 @@ export default class ReportEmail extends Report { } } + if (Object.keys(this.templates).length) { + this.enabled = false; + this.logger.log({ + level: 'error', + message: "Email templates cannot be associated to channels." + }); + } + setInterval(() => { const nextEmail = this.emailBacklog.pop(); if (nextEmail) { @@ -78,7 +78,14 @@ export default class ReportEmail extends Report { } }, 3000); } - } + }; + + + getUserGroup = (group) => { + const groups = this.params.notifiedEmails || this.params.userGroups; + + return groups[group] || groups["default"]; + }; getEmails = (content) => { const users = content.data @@ -92,13 +99,9 @@ export default class ReportEmail extends Report { .filter(item => !!item); try { - const emails = [...new Set(users)] - .map(user => { - return this.params.notifiedEmails[user]; - }) + return [...new Set(users)] + .map(user => this.getUserGroup(user)) .filter(item => !!item); - - return (emails.length) ? emails : [this.params.notifiedEmails["default"]]; } catch (error) { this.logger.log({ level: 'error', @@ -117,24 +120,19 @@ export default class ReportEmail extends Report { }; _sendEmail = (email) => { - if (this.transporter) { - this.transporter - .sendMail(email) - .catch(error => { - this.logger.log({ - level: 'error', - message: error - }); - }) - } + this.transporter + .sendMail(email) + .catch(error => { + this.logger.log({ + level: 'error', + message: error + }); + }); }; report = (channel, content) => { - if (Object.keys(this.templates).length > 0 && - this.params.notifiedEmails && - this.params.notifiedEmails["default"] && - this.params.notifiedEmails["default"].length) { + if (this.enabled) { const emailGroups = this.getEmails(content); for (let emails of emailGroups) { diff --git a/src/reports/reportHTTP.js b/src/reports/reportHTTP.js index 5189be5..60c8105 100644 --- a/src/reports/reportHTTP.js +++ b/src/reports/reportHTTP.js @@ -39,19 +39,13 @@ export default class ReportHTTP extends Report { this.name = "reportHTTP" || this.params.name; this.enabled = true; - if (!this.params.hooks || !Object.keys(this.params.hooks).length){ + + if (!this.getUserGroup("default")) { this.logger.log({ level: 'error', - message: `${this.name} reporting is not enabled: no group is defined` + message: `${this.name} reporting is not enabled: no default group defined` }); this.enabled = false; - } else { - if (!this.params.hooks["default"]) { - this.logger.log({ - level: 'error', - message: `In hooks, for ${this.name}, a group named 'default' is required for communications to the admin.` - }); - } } this.headers = this.params.headers || {}; @@ -60,12 +54,18 @@ export default class ReportHTTP extends Report { } } + getUserGroup = (group) => { + const groups = this.params.hooks || this.params.userGroups; + + return groups[group] || groups["default"]; + }; + getTemplate = (group, channel, content) => { return this.params.templates[channel] || this.params.templates["default"]; }; _sendHTTPMessage = (group, channel, content) => { - const url = this.params.hooks[group] || this.params.hooks["default"]; + const url = this.getUserGroup(group); if (url) { const context = this.getContext(channel, content); @@ -93,7 +93,7 @@ export default class ReportHTTP extends Report { if (this.enabled) { let groups = content.data.map(i => i.matchedRule.group).filter(i => i != null); - groups = (groups.length) ? [...new Set(groups)] : Object.keys(this.params.hooks); // If there is no specific group defined, send to all of them + groups = (groups.length) ? [...new Set(groups)] : [this.getUserGroup("default")]; for (let group of groups) { this._sendHTTPMessage(group, channel, content); diff --git a/src/reports/reportSyslog.js b/src/reports/reportSyslog.js index 781d46d..6b98430 100644 --- a/src/reports/reportSyslog.js +++ b/src/reports/reportSyslog.js @@ -48,7 +48,6 @@ export default class ReportSyslog extends Report { }; } - _getMessage = (channel, content) => { return this.parseTemplate(this.params.templates[channel] || this.params.templates["default"], this.getContext(channel, content)); }; diff --git a/src/reports/reportWebex.js b/src/reports/reportWebex.js index ca62fb2..ed11405 100644 --- a/src/reports/reportWebex.js +++ b/src/reports/reportWebex.js @@ -38,22 +38,20 @@ export default class ReportWebex extends Report { super(channels, params, env); - this.enabled = true; - if (!this.params.hooks || !Object.keys(this.params.hooks).length){ + if (!this.getUserGroup("default")) { this.logger.log({ level: 'error', - message: "Webex reporting is not enabled: no group is defined" + message: `Webex reporting is not enabled: no default group defined` }); this.enabled = false; - } else { - if (!this.params.hooks["default"]) { - this.logger.log({ - level: 'error', - message: "In hooks, for reportWebex, a group named 'default' is required for communications to the admin." - }); - } } - } + }; + + getUserGroup = (group) => { + const groups = this.params.hooks || this.params.userGroups; + + return groups[group] || groups["default"]; + }; _sendWebexMessage = (url, message, content) => { @@ -77,7 +75,7 @@ export default class ReportWebex extends Report { if (this.enabled){ let groups = content.data.map(i => i.matchedRule.group).filter(i => i != null); - groups = (groups.length) ? [...new Set(groups)] : Object.keys(this.params.hooks); // If there is no specific group defined, send to all of them + groups = (groups.length) ? [...new Set(groups)] : [this.getUserGroup("default")]; for (let group of groups) { if (this.params.hooks[group]) { diff --git a/src/worker.js b/src/worker.js index 1314498..80db8af 100644 --- a/src/worker.js +++ b/src/worker.js @@ -30,23 +30,25 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import Input from "./inputs/inputYml"; import cluster from "cluster"; import fs from "fs"; +import inputYml from "./inputs/inputYml"; // Default input connector export default class Worker { - constructor(configFile, volume) { + constructor({ configFile, volume, configConnector, inputConnector, groupFile }) { + global.EXTERNAL_CONFIG_CONNECTOR = global.EXTERNAL_CONFIG_CONNECTOR || configConnector; + global.EXTERNAL_INPUT_CONNECTOR = global.EXTERNAL_INPUT_CONNECTOR || inputConnector; global.EXTERNAL_CONFIG_FILE = global.EXTERNAL_CONFIG_FILE || configFile; + global.EXTERNAL_GROUP_FILE = global.EXTERNAL_GROUP_FILE || groupFile; global.EXTERNAL_VOLUME_DIRECTORY = global.EXTERNAL_VOLUME_DIRECTORY || volume; const env = require("./env"); this.config = env.config; this.logger = env.logger; - this.input = new Input(env); + this.input = new (global.EXTERNAL_INPUT_CONNECTOR || inputYml)(env); this.pubSub = env.pubSub; this.version = env.version; - this.configFile = env.configFile; if (!this.config.multiProcess) { const Consumer = require("./consumer").default; @@ -70,7 +72,6 @@ export default class Worker { const ConnectorFactory = require("./connectorFactory").default; console.log("BGPalerter, version:", this.version, "environment:", this.config.environment); - console.log("Loaded config:", this.configFile); // Write pid on a file if (this.config.pidFile) { diff --git a/tests/1_config.js b/tests/1_config.js index 82cc1a5..24bd98a 100644 --- a/tests/1_config.js +++ b/tests/1_config.js @@ -48,6 +48,7 @@ if (!fs.existsSync(volume)) { } fs.copyFileSync("tests/config.test.yml", volume + "config.test.yml"); fs.copyFileSync("tests/prefixes.test.yml", volume + "prefixes.test.yml"); +fs.copyFileSync("tests/groups.test.yml", volume + "groups.test.yml"); describe("Core functions", function() { @@ -74,6 +75,7 @@ describe("Core functions", function() { "fadeOffSeconds", "checkFadeOffGroupsSeconds", "volume", + "groupsFile", "persistStatus", "rpki" ]); diff --git a/tests/4_groups.js b/tests/4_groups.js new file mode 100644 index 0000000..4543416 --- /dev/null +++ b/tests/4_groups.js @@ -0,0 +1,84 @@ +/* + * BSD 3-Clause License + * + * Copyright (c) 2019, NTT Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +const chai = require("chai"); +const fs = require("fs"); +const chaiSubset = require('chai-subset'); +chai.use(chaiSubset); +const expect = chai.expect; +const volume = "volumetests/"; +const asyncTimeout = 20000; + +// Prepare test environment +if (!fs.existsSync(volume)) { + fs.mkdirSync(volume); +} +fs.copyFileSync("tests/config.test.yml", volume + "config.test.yml"); +fs.copyFileSync("tests/prefixes.test.yml", volume + "prefixes.test.yml"); +fs.copyFileSync("tests/groups.test.yml", volume + "groups.test.yml"); + +global.EXTERNAL_CONFIG_FILE = volume + "config.test.yml"; + +const worker = require("../index"); + +describe("External groups file", function() { + + it("load groups", function () { + const config = worker.config; + expect(config.groupsFile).to.equal("groups.test.yml"); + expect(config.reports[0].params.userGroups).to + .containSubset({ + test: [ + "filename" + ] + }); + }) + .timeout(asyncTimeout); + + it("watch groups", function (done) { + + fs.copyFileSync("tests/groups.test.after.yml", "volumetests/groups.test.yml"); + + setTimeout(() => { + const config = worker.config; + expect(config.reports[0].params.userGroups).to + .containSubset({ + test: [ + "filename-after" + ] + }); + done(); + }, 10000); + + }) + .timeout(asyncTimeout); +}); \ No newline at end of file diff --git a/tests/config.test.yml b/tests/config.test.yml index 20a8e68..7ff2224 100644 --- a/tests/config.test.yml +++ b/tests/config.test.yml @@ -79,6 +79,8 @@ persistStatus: true volume: volumetests/ +groupsFile: groups.test.yml + processMonitors: - file: uptimeApi params: @@ -86,7 +88,6 @@ processMonitors: host: null port: 8011 - rpki: vrpProvider: ntt preCacheROAs: true diff --git a/tests/groups.test.after.yml b/tests/groups.test.after.yml new file mode 100644 index 0000000..f41e16c --- /dev/null +++ b/tests/groups.test.after.yml @@ -0,0 +1,4 @@ + +reportFile: + test: + - filename-after \ No newline at end of file diff --git a/tests/groups.test.yml b/tests/groups.test.yml new file mode 100644 index 0000000..5db79ce --- /dev/null +++ b/tests/groups.test.yml @@ -0,0 +1,4 @@ + +reportFile: + test: + - filename \ No newline at end of file diff --git a/tests/npm_tests/testNpmLib.js b/tests/npm_tests/testNpmLib.js new file mode 100644 index 0000000..3faf8f5 --- /dev/null +++ b/tests/npm_tests/testNpmLib.js @@ -0,0 +1,66 @@ +/* + * BSD 3-Clause License + * + * Copyright (c) 2019, NTT Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +const chai = require("chai"); +const chaiSubset = require('chai-subset'); +chai.use(chaiSubset); +const expect = chai.expect; +const volume = "volumetests/"; +const asyncTimeout = 20000; + +const Config = require("../../src/config/config").default; + +const ConfigTest = function () { + + this.retrieve = () => { + const data = (new Config()).default; + + data.test = true; + + return data; + } + + this.save = () => { + return true; + } +}; + +describe("External Connector", function() { + + it("load external connector", function () { + const Worker = require("../../src/worker").default; + const worker = new Worker({ volume, configConnector: ConfigTest }); + const config = worker.config; + expect(config.test).to.equal(true); + }) + .timeout(asyncTimeout); +}); \ No newline at end of file