I have a setup to push all my rails application logs to cloud watch log groups,Now I need to run a lambda function daily at the end of the day to parse all the logs from cloudwatch log groups and count number of 500 error codes in that day and has to trigger an SNS notification.
currently I have a setup, I created the metric filter on log groups to filter the logs based on a keyword, whenever the metric filter finds the keyword , it will raise an alarm and triggers an lambda function, this lambda function takes the timezone of that keyword and collects logs 5 seconds before and 5 seconds after and triggers an another sns notification to send an email.
Below is the NodeJs code which will perform the above mentioned activity.
var aws = require('aws-sdk');
var cwl = new aws.CloudWatchLogs();
var sns = new aws.SNS();
exports.handler = function(event, context) {
console.log(event);
var message = JSON.parse(event.Records[0].Sns.Message);
var requestParams = {
metricName: message.Trigger.MetricName,
metricNamespace: message.Trigger.Namespace
};
cwl.describeMetricFilters(requestParams, function(err, data) {
if(err) console.log('Error is:', err);
else {
console.log('Metric Filter data is:', data);
getLogsAndSendEmail(message, data);
}
});
};
function getLogsAndSendEmail(message, metricFilterData) {
var metricFilter = metricFilterData.metricFilters[0];
let runningCount=4;
var params = {
logGroupName: metricFilter.logGroupName,
descending: true,
limit: runningCount,
orderBy: 'LastEventTime'
};
cwl.describeLogStreams(params, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else{
let logStreams=new Array();
for ( var i = 0; i < runningCount; i++ ) {
logStreams[i]= data.logStreams[i].logStreamName;
}
var parameters = {
'logGroupName' : metricFilter.logGroupName,
'logStreamNames': logStreams,
'filterPattern' : metricFilter.filterPattern ? metricFilter.filterPattern : '"Internal error"',
'startTime' : (new Date()).getTime()-60*1000
};
cwl.filterLogEvents(parameters, function (err, data){
if (err) {
console.log('Filtering failure:', err);
} else {
console.log(data.events.length)
console.log('above is data.events.length')
for (var i = 0; i < data.events.length; i++) {
getDetailedLog(data.events[i],metricFilter.logGroupName);
}
console.log('Exiting Describe Logs ');
}
});
}
});
}
function getDetailedLog(item,logGroupName){
console.log('Entering Detailed Logs ');
var timestamp=item.timestamp;
var parameters2 = {
'logGroupName' : logGroupName,
'logStreamNames': [item.logStreamName],
'startTime' : timestamp - 5000,
'endTime' : timestamp + 5000,
};
cwl.filterLogEvents(parameters2, function (err, data){
if (err) {
console.log('Filtering failure:', err);
} else {
console.log("===SENDING TO SNS===");
let errorLogs=data.events.map(x => x.message);
sendToSNS(errorLogs,logGroupName);
}
});
}
function sendToSNS(data,logGroupName){
var logData = 'LogGroupName : '+logGroupName+'\n\nError Logs: \n\n';
for ( var i=0;i<data.length;i++ ) {
logData += ( data[i] + '\n\n');
}
var snsEmailArn='arn:aws:sns:ap-southeast-1:test-logs-topic';
var params = {
Message: logData,
Subject: 'error LOGS',
TopicArn: snsEmailArn
};
sns.publish(params, function(err, data) {
if (err) {
console.log(err, err.stack); // an error occurred
}
else {
console.log("SNS email success: "); // successful response
}
});
}
Now my requirement is to parse the logs and count number of 500 internal server errors that are logged in this cloudwatch log group in a day and triggers a notification to send an email regarding this information on daily basis.
please help me in editing this code and in meeting my requirement.
