mirror/nttgin-BGPalerter
1
0
Fork 0
mirror of https://github.com/nttgin/BGPalerter.git synced 2024-05-19 06:50:08 +00:00
Code Issues Releases Wiki Activity

introduced possibility to re-generate prefix list periodically

Browse Source
This commit is contained in:
Massimo Candela
2020-08-03 16:00:08 +02:00
parent 0fa6fd6f4b
commit e2bb8d5e5a
20 changed files with 164 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
config.yml.example
View File

@@ -223,7 +223,7 @@ reports:
# Persist the status of BGPalerter. If the process is restarted, the list of alerts already sent is recovered
# and they are not repeated. The process must be able to write on disc, this option will create a file inside .cache/
notificationIntervalSeconds: 14400
notificationIntervalSeconds: 86400
persistStatus: true
logging:
@@ -235,6 +235,7 @@ logging:
compressOnRotation: false
checkForUpdatesAtBoot: true
generatePrefixListEveryDays: 2
############################
# Process monitoring settings:

4
docs/configuration.md
View File

@@ -18,7 +18,9 @@ The following are common parameters which it is possible to specify in the confi
|processMonitors| A list of modules allowing various ways to check for the status of BGPalerter (e.g. API, heartbeat). See [here](process-monitors.md) for more information. | | | No |
|httpProxy| Defines the HTTP/HTTPS proxy server to be used by BGPalerter and its submodules (reporters/connectors/monitors). See [here](http-proxy.md) for more information. | A string | http://usr:psw@ prxy.org:8080 | No |
|volume| Defines a directory that will contain the data that needs persistence. For example, configuration files and logs will be created in such directory (default to "./"). | A string | /home/bgpalerter/ | No |
|persistStatus| If set to true, when BGPalerter is restarted the list of alerts already sent is recovered. This avoids duplicated alerts. The process must be able to write on disc inside `.cache/`. | A boolean | true | No |
|persistStatus| If set to true, when BGPalerter is restarted the list of alerts already sent is recovered. This avoids duplicated alerts. The process must be able to write on disc inside `.cache/`. | A boolean | true | No |
|generatePrefixListEveryDays| This parameter allows to automatically re-generate the prefix list after the specified amount of days. | An integer | 2 | No |
The following are advanced parameters, please don't touch them if you are not doing research/experiments.

3
index.js
View File

@@ -162,7 +162,8 @@ switch(params._[0]) {
debug,
historical,
group: params.g,
append: !!params.A
append: !!params.A,
logger: null
};
generatePrefixes(inputParameters);

13
src/consumer.js
View File

@@ -30,11 +30,10 @@
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
import env from "./env";
export default class Consumer {
constructor(){
constructor(env, input){
this.logger = env.logger;
this.connectors = {};
for (let connector of env.config.connectors) {
@@ -44,13 +43,13 @@ export default class Consumer {
try {
this.monitors = env.config.monitors
.map(monitor => new monitor.class(monitor.name, monitor.channel, monitor.params || {}, env));
.map(monitor => new monitor.class(monitor.name, monitor.channel, monitor.params || {}, env, input));
this.reports = env.config.reports
.map(report => new report.class(report.channels, report.params || {}, env));
} catch (error) {
env.logger.log({
this.logger.log({
level: 'error',
message: error
});
@@ -79,7 +78,7 @@ export default class Consumer {
monitor
.monitor(message)
.catch(error => {
env.logger.log({
this.logger.log({
level: 'error',
message: error
});
@@ -88,7 +87,7 @@ export default class Consumer {
}
}
} catch (error) {
env.logger.log({
this.logger.log({
level: 'error',
message: error.message
});

13
src/env.js
View File

@@ -35,8 +35,8 @@ import fs from "fs";
import path from "path";
import PubSub from './utils/pubSub';
import FileLogger from './utils/fileLogger';
import Input from "./inputs/inputYml";
import {version} from '../package.json';
import Storage from './utils/storages/storageFile';
import axios from 'axios';
import url from 'url';
@@ -299,18 +299,9 @@ if (config.httpProxy) {
vector.agent = new HttpsProxyAgent(url.parse(config.httpProxy));
}
if (!!config.persistStatus) {
const Storage = require("./utils/storages/storageFile").default;
vector.storage = new Storage({
validitySeconds: config.notificationIntervalSeconds,
}, config);
}
const input = new Input(config);
vector.storage = new Storage({}, config);
vector.config = config;
vector.logger = wlogger;
vector.input = input;
vector.pubSub = new PubSub();
module.exports = vector;

62
src/generatePrefixesList.js
View File

@@ -22,9 +22,12 @@ module.exports = function generatePrefixes(inputParameters) {
debug,
historical,
group,
append
append,
logger
} = inputParameters;
logger = logger || console.log;
const generateList = {};
const allOrigins = {};
let someNotValidatedPrefixes = false;
@@ -35,7 +38,7 @@ module.exports = function generatePrefixes(inputParameters) {
}
if (historical) {
console.log("WARNING: you are using historical visibility data for generating the prefix list.");
logger("WARNING: you are using historical visibility data for generating the prefix list.");
}
if (!asnList && !prefixes) {
@@ -73,7 +76,7 @@ module.exports = function generatePrefixes(inputParameters) {
});
if (debug) {
console.log("Query", url)
logger("Query", url)
}
return axios({
@@ -91,13 +94,13 @@ module.exports = function generatePrefixes(inputParameters) {
return asns;
})
.catch((error) => {
console.log(error);
console.log("RIPEstat prefix-overview query failed: cannot retrieve information for " + prefix);
logger(error);
logger(`RIPEstat prefix-overview query failed: cannot retrieve information for ${prefix}`);
});
};
const getAnnouncedMoreSpecifics = (prefix) => {
console.log("Generating monitoring rule for", prefix);
logger(`Generating monitoring rule for ${prefix}`);
const url = brembo.build("https://stat.ripe.net", {
path: ["data", "related-prefixes", "data.json"],
params: {
@@ -107,7 +110,7 @@ module.exports = function generatePrefixes(inputParameters) {
});
if (debug) {
console.log("Query", url)
logger(`Query ${url}`);
}
return axios({
@@ -122,7 +125,7 @@ module.exports = function generatePrefixes(inputParameters) {
prefixes = data.data.data.prefixes
.filter(i => i.relationship === "Overlap - More Specific")
.map(i => {
console.log("Detected more specific " + i.prefix);
logger(`Detected more specific ${i.prefix}`);
return {
asn: i.origin_asn,
description: i.asn_name,
@@ -134,7 +137,7 @@ module.exports = function generatePrefixes(inputParameters) {
return prefixes;
})
.catch(() => {
console.log("RIPEstat related-prefixes query failed: cannot retrieve information for " + prefix);
logger(`RIPEstat related-prefixes query failed: cannot retrieve information for ${prefix}`);
});
};
@@ -169,10 +172,10 @@ module.exports = function generatePrefixes(inputParameters) {
}
});
console.log(`Getting announced prefixes of AS${asn}`);
logger(`Getting announced prefixes of AS${asn}`);
if (debug) {
console.log("Query", url)
logger(`Query ${url}`);
}
return axios({
@@ -202,7 +205,7 @@ module.exports = function generatePrefixes(inputParameters) {
})
.then(list => {
if (list.length === 0) {
console.log(`WARNING: no announced prefixes were detected for AS${asn}. If you are sure the AS provided is announcing at least one prefix, this could be an issue with the data source (RIPEstat). Try to run the generate command with the option -H.`);
logger(`WARNING: no announced prefixes were detected for AS${asn}. If you are sure the AS provided is announcing at least one prefix, this could be an issue with the data source (RIPEstat). Try to run the generate command with the option -H.`);
}
return list;
})
@@ -227,14 +230,14 @@ module.exports = function generatePrefixes(inputParameters) {
// All good
} else if (isValid === false) {
delete generateList[prefix];
console.log("RPKI invalid:", prefix, asn);
logger(`RPKI invalid: ${prefix} ${asn}`);
} else {
generateList[prefix].description += ' (No ROA available)';
someNotValidatedPrefixes = true;
}
})
.catch((error) => {
console.log("RPKI validation query failed: cannot retrieve information for " + prefix);
logger(`RPKI validation query failed: cannot retrieve information for ${prefix}`);
});
};
@@ -252,7 +255,7 @@ module.exports = function generatePrefixes(inputParameters) {
.then(plist => prefixes = prefixes.concat(plist));
})
.then(() => {
console.log(`Total prefixes detected: ${prefixes.length}`);
logger(`Total prefixes detected: ${prefixes.length}`);
return prefixes;
});
}
@@ -297,18 +300,18 @@ module.exports = function generatePrefixes(inputParameters) {
});
})
.catch((e) => {
console.log("Cannot download more specific prefixes of", prefix, e);
logger(`Cannot download more specific prefixes of ${prefix} ${e}`);
})
})
.catch((e) => {
console.log("Cannot download more specific prefixes", e);
logger(`Cannot download more specific prefixes ${e}`);
})
})
.then(() => rpki.preCache())
.then(() => { // Check
return Promise.all(Object.keys(generateList).map(prefix => validatePrefix(generateList[prefix].asn[0], prefix)))
.catch((e) => {
console.log("ROA check failed due to error", e);
logger(`ROA check failed due to error ${e}`);
})
})
.then(() => { // Add the options for monitorASns
@@ -317,7 +320,7 @@ module.exports = function generatePrefixes(inputParameters) {
generateList.options = generateList.options || {};
generateList.options.monitorASns = generateList.options.monitorASns || {};
for (let monitoredAs of list) {
console.log("Generating generic monitoring rule for AS", monitoredAs);
logger(`Generating generic monitoring rule for AS${monitoredAs}`);
generateList.options.monitorASns[monitoredAs] = {
group: group || "default"
};
@@ -330,22 +333,21 @@ module.exports = function generatePrefixes(inputParameters) {
}
// Otherwise nothing
})
.then(() => {
if (someNotValidatedPrefixes) {
logger("WARNING: the generated configuration is a snapshot of what is currently announced. Some of the prefixes don't have ROA objects associated or are RPKI invalid. Please, verify the config file by hand!");
}
})
.then(() => { // write everything into the file
if (append) {
const finalList = getCurrentPrefixes(outputFile, generateList);
fs.writeFileSync(outputFile, yaml.dump(finalList));
} else {
fs.writeFileSync(outputFile, yaml.dump(generateList));
}
const list = (append) ? getCurrentPrefixes(outputFile, generateList) : generateList;
if (someNotValidatedPrefixes) {
console.log("WARNING: the generated configuration is a snapshot of what is currently announced. Some of the prefixes don't have ROA objects associated or are RPKI invalid. Please, verify the config file by hand!");
}
console.log("Done!");
fs.writeFileSync(outputFile, yaml.dump(list));
logger("Done!");
})
.catch((e) => {
console.log("Something went wrong", e);
logger(`Something went wrong ${e}`);
})
};

78
src/inputs/input.js
View File

@@ -33,18 +33,22 @@
import ipUtils from "ip-sub";
import inquirer from "inquirer";
import generatePrefixes from "../generatePrefixesList";
export default class Input {
constructor(config){
constructor(env){
this.prefixes = [];
this.asns = [];
this.cache = {
af: {},
binaries: {}
};
this.config = config;
this.config = env.config;
this.storage = env.storage;
this.logger = env.logger;
this.callbacks = [];
this.prefixListStorageKey = 'generate-prefixes-config';
setTimeout(() => {
this.loadPrefixes()
@@ -52,11 +56,16 @@ export default class Input {
this._change();
})
.catch(error => {
this.logger.log({
level: 'error',
message: error
});
console.log(error);
process.exit();
});
}, 200);
this.setReGeneratePrefixList();
};
_isAlreadyContained = (prefix, lessSpecifics) => {
@@ -193,7 +202,7 @@ export default class Input {
}
])
.then((answer) => {
const generatePrefixes = require("../generatePrefixesList");
// const generatePrefixes = require("../generatePrefixesList");
const asns = answer.asns.split(",");
const inputParameters = {
@@ -207,10 +216,24 @@ export default class Input {
debug: false,
historical: false,
group: null,
append: false
append: false,
logger: null
};
if (this.config.generatePrefixListEveryDays >= 1) {
return this.storage
.set(this.prefixListStorageKey, inputParameters)
.then(() => generatePrefixes(inputParameters))
.catch(error => {
this.logger.log({
level: 'error',
message: error
});
});
} else {
return generatePrefixes(inputParameters);
}
return generatePrefixes(inputParameters);
});
} else {
throw new Error("Nothing to monitor.");
@@ -220,4 +243,49 @@ export default class Input {
};
_reGeneratePrefixList = () => {
this.logger.log({
level: 'info',
message: "Updating prefix list"
});
this.storage
.get(this.prefixListStorageKey)
.then(inputParameters => {
inputParameters.logger = (message) => {
this.logger.log({
level: 'info',
message
});
};
return generatePrefixes(inputParameters);
})
.then(() => {
this.logger.log({
level: 'info',
message: "Prefix list updated. See prefixes.yml."
});
this.setReGeneratePrefixList();
})
.catch(error => {
this.logger.log({
level: 'error',
message: error
});
});
};
setReGeneratePrefixList = () => {
if (this.config.generatePrefixListEveryDays >= 1) {
//const refreshTimer = Math.ceil(this.config.generatePrefixListEveryDays) * 24 * 3600 * 1000;
const refreshTimer = 60000;
if (this.regeneratePrefixListTimer) {
clearTimeout(this.regeneratePrefixListTimer);
}
this.regeneratePrefixListTimer = setTimeout(this._reGeneratePrefixList, refreshTimer);
}
};
}

11
src/inputs/inputYml.js
View File

@@ -38,12 +38,12 @@ import { AS } from "../model";
export default class InputYml extends Input {
constructor(config){
super(config);
constructor(env){
super(env);
this.prefixes = [];
this.asns = [];
if (!config.monitoredPrefixesFiles || config.monitoredPrefixesFiles.length === 0) {
if (!this.config.monitoredPrefixesFiles || this.config.monitoredPrefixesFiles.length === 0) {
throw new Error("The monitoredPrefixesFiles key is missing in the config file");
}
};
@@ -66,7 +66,10 @@ export default class InputYml extends Input {
return this._change();
})
.catch(error => {
console.log(error);
this.logger.log({
level: 'error',
message: error
});
process.exit();
});
});

8
src/monitors/monitor.js
View File

@@ -35,11 +35,11 @@ import axios from "axios";
export default class Monitor {
constructor(name, channel, params, env) {
constructor(name, channel, params, env, input) {
this.config = env.config;
this.pubSub = env.pubSub;
this.logger = env.logger;
this.input = env.input;
this.input = input;
this.storage = env.storage;
this.params = params || {};
this.maxDataSamples = this.params.maxDataSamples || 1000;
@@ -163,7 +163,7 @@ export default class Monitor {
};
_retrieveStatus = () => {
if (this.storage) {
if (this.config.persistStatus && this.storage) {
this.storage
.get(`status-${this.name}`)
.then(({ alerts={}, sent={}, truncated={}, fadeOff={} }) => {
@@ -189,7 +189,7 @@ export default class Monitor {
};
_persistStatusHelper = () => {
if (this.storage) {
if (this.config.persistStatus && this.storage) {
const status = {
alerts: this.alerts,
sent: this.sent,

4
src/monitors/monitorAS.js
View File

@@ -34,8 +34,8 @@ import Monitor from "./monitor";
export default class MonitorAS extends Monitor {
constructor(name, channel, params, env){
super(name, channel, params, env);
constructor(name, channel, params, env, input){
super(name, channel, params, env, input);
this.thresholdMinPeers = (params && params.thresholdMinPeers != null) ? params.thresholdMinPeers : 3;
this.updateMonitoredResources();
};

4
src/monitors/monitorHijack.js
View File

@@ -35,8 +35,8 @@ import ipUtils from "ip-sub";
export default class MonitorHijack extends Monitor {
constructor(name, channel, params, env){
super(name, channel, params, env);
constructor(name, channel, params, env, input){
super(name, channel, params, env, input);
this.thresholdMinPeers = (params && params.thresholdMinPeers != null) ? params.thresholdMinPeers : 2;
this.updateMonitoredResources();
};

4
src/monitors/monitorNewPrefix.js
View File

@@ -34,8 +34,8 @@ import Monitor from "./monitor";
export default class MonitorNewPrefix extends Monitor {
constructor(name, channel, params, env){
super(name, channel, params, env);
constructor(name, channel, params, env, input){
super(name, channel, params, env, input);
this.thresholdMinPeers = (params && params.thresholdMinPeers != null) ? params.thresholdMinPeers : 3;
this.updateMonitoredResources();
};

4
src/monitors/monitorPassthrough.js
View File

@@ -2,8 +2,8 @@ import Monitor from "./monitor";
export default class monitorPassthrough extends Monitor {
constructor(name, channel, params, env){
super(name, channel, params, env);
constructor(name, channel, params, env, input){
super(name, channel, params, env, input);
this.count = 0;
};

4
src/monitors/monitorPath.js
View File

@@ -34,8 +34,8 @@ import Monitor from "./monitor";
export default class MonitorPath extends Monitor {
constructor(name, channel, params, env){
super(name, channel, params, env);
constructor(name, channel, params, env, input){
super(name, channel, params, env, input);
this.thresholdMinPeers = (params && params.thresholdMinPeers != null) ? params.thresholdMinPeers : 1;
this.updateMonitoredResources();
};

4
src/monitors/monitorRPKI.js
View File

@@ -4,8 +4,8 @@ import fs from "fs";
export default class MonitorRPKI extends Monitor {
constructor(name, channel, params, env){
super(name, channel, params, env);
constructor(name, channel, params, env, input){
super(name, channel, params, env, input);
this.providers = [ "ntt", "ripe", "external"]; // First provider is the default one

4
src/monitors/monitorSwUpdates.js
View File

@@ -34,8 +34,8 @@ import Monitor from "./monitor";
export default class MonitorSwUpdates extends Monitor {
constructor(name, channel, params, env){
super(name, channel, params, env);
constructor(name, channel, params, env, input){
super(name, channel, params, env, input);
};
updateMonitoredResources = () => {

4
src/monitors/monitorVisibility.js
View File

@@ -35,8 +35,8 @@ import ipUtils from "ip-sub";
export default class MonitorVisibility extends Monitor {
constructor(name, channel, params, env){
super(name, channel, params, env);
constructor(name, channel, params, env, input){
super(name, channel, params, env, input);
this.thresholdMinPeers = (params && params.thresholdMinPeers != null) ? params.thresholdMinPeers : 40;
if (params.threshold) {
this.logger.log({

2
src/utils/storage.js
View File

@@ -3,7 +3,7 @@ export default class Storage {
constructor(params, config){
this.config = config;
this.params = params;
this.validity = (this.params.validitySeconds || 3600 * 2) * 1000;
this.validity = (this.params.validitySeconds ? (this.params.validitySeconds * 1000) : null) || Infinity;
};
set = (key, value) => {

18
src/worker.js
View File

@@ -30,7 +30,7 @@
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
import LossyBuffer from "./utils/lossyBuffer";
import Input from "./inputs/inputYml";
import cluster from "cluster";
import fs from "fs";
@@ -40,31 +40,35 @@ export default class Worker {
global.EXTERNAL_VOLUME_DIRECTORY = global.EXTERNAL_VOLUME_DIRECTORY || volume;
const env = require("./env");
const Consumer = require("./consumer").default;
this.config = env.config;
this.logger = env.logger;
this.input = env.input;
this.input = new Input(env);
this.pubSub = env.pubSub;
this.version = env.version;
this.configFile = env.configFile;
if (!this.config.multiProcess) {
const Consumer = require("./consumer").default;
this.master();
new Consumer();
new Consumer(env, this.input);
} else {
if (cluster.isMaster) {
this.master(cluster.fork());
} else {
new Consumer();
const Consumer = require("./consumer").default;
new Consumer(env, this.input);
}
}
};
master = (worker) => {
const LossyBuffer = require("./utils/lossyBuffer").default;
const ConnectorFactory = require("./connectorFactory").default;
console.log("BGPalerter, version:", this.version, "environment:", this.config.environment);
console.log("Loaded config:", this.configFile);
@@ -80,7 +84,6 @@ export default class Worker {
}
}
const ConnectorFactory = require("./connectorFactory").default;
const connectorFactory = new ConnectorFactory();
if (this.config.uptimeMonitor) {
@@ -101,7 +104,8 @@ export default class Worker {
this.config.maxMessagesPerSecond = this.config.maxMessagesPerSecond || 6000;
const buffer = new LossyBuffer(parseInt(this.config.maxMessagesPerSecond /(1000/bufferCleaningInterval)), bufferCleaningInterval, this.logger);
connectorFactory.loadConnectors();
return connectorFactory.connectConnectors(this.input)
return connectorFactory
.connectConnectors(this.input)
.then(() => {
for (const connector of connectorFactory.getConnectors()) {

9
tests/generate_tests/tests.js
View File

@@ -59,9 +59,9 @@ describe("Prefix List", function() {
debug: false,
historical: false,
group: null,
append: false
append: false,
logger: () => {}
}
console.log = function(){};
generatePrefixes(inputParameters)
.then(() => {
const result = fs.readFileSync(outputFile, 'utf8');
@@ -94,9 +94,10 @@ describe("Prefix List", function() {
debug: false,
historical: false,
group: "test",
append: false
append: false,
logger: () => {}
}
console.log = function(){};
generatePrefixes(inputParameters)
.then(() => {
const result = fs.readFileSync(outputFile, 'utf8');