From 9f1cbc18c6655b4d03f100cf6ba054cbc4350042 Mon Sep 17 00:00:00 2001 From: David Lang Date: Fri, 2 Nov 2012 15:14:50 -0700 Subject: [PATCH 01/13] Fixed many problems in sending messages A prior commit had changed the files array to a two-level array $file[fileid][tailinfo,productname] This broke the File::Tail interface that was passed a reference to @file so this change was reverted, the array @names was created to hold the product names, and @pending was created to hold the prior data for multi-line logs many debug prints were put in, but they are all commented out. In addition, loglog will now output any pending data when a new timeslice rolls around and there is no new data to send. --- loglog | 65 +++++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/loglog b/loglog index fb5baba..fba1522 100755 --- a/loglog +++ b/loglog @@ -5,6 +5,7 @@ use Time::localtime; use vars qw($VERSION); use IO::Socket; use Sys::Hostname; +use Data::Dumper; $|=1; # Note that Proc::Daemon, and File::Tail are included inline. @@ -85,6 +86,11 @@ $host = `hostname -s`; chomp($host); $conf = "/opt/disys/etc/loglog/$host.conf"; # print "config is $conf\n"; +#these three arrays are tied together, element X of each array correspond to each other. +@files=(); +@names=(); +@pending=(); + my $timestamp = localtime(time); sendlog("status","Startup (program timestamp is $mytimestamp)"); @@ -113,13 +119,15 @@ open (I, $conf); while () { chomp; ($filename,$product) = split(" ",$_); - push(@files, [File::Tail->new(name=>"$filename",debug=>$debug),$product]); + if ($product eq '') {$product = "unknown";} + push(@files, File::Tail->new(name=>"$filename",debug=>$debug)); + push(@names, $product); } close I; my $rin=''; $interval = 5 * 60; # 5 minutes -$sent=0; $sentbytes=0; $sentmaxlength=0; $nextreport += $interval; +$sent=0; $sentbytes=0; $sentmaxlength=0; $nextreport = time + $interval; sendlog("status","finished init, configured to watch ".($#files+1)." logfiles, starting processing of logs"); while ($continue) { @@ -127,31 +135,48 @@ sendlog("status","finished init, configured to watch ".($#files+1)." logfiles, s unless ($nfound) { my @ints; foreach(@files) { - push(@ints,$_[0]->interval); + push(@ints,$_->interval); } } - foreach (@files) { - if ($_[0]->predict == 0) { - my $t = $_[0]->read; chomp($t); - my $name = $_[0]->{"input"}; + for($i=0; $i<=$#files; $i++) { + #print "$i $names[$i]\n"; + #print " ".time."\n"; + $thisfiletail = $files[$i]; + if ($thisfiletail->predict == 0) { + my $t = $thisfiletail->read; chomp($t); + #print " $t\n"; + my $name = $thisfiletail->{"input"}; if ($t =~ /^\s/ ) { # the line starts with whitespace, assume it's a continuation of the prior line # append it to the prior line, include a separator to indicate that we have done so - $_[0]->pending .= " #015 ".$t; + $pending[$i] .= " #015 ".$t; } else { # if there is a prior line send it - if ($_[0]->pending ne "") { - $filename = $_[0]->{"input"}; - $product = $_[1] || "unknown"; - sendlog($product, $_[0]->pending); + if ($pending[$i] ne "") { + $filename = $thisfiletail->{"input"}; + $product = $$names[$i]; + sendlog($product, $pending[$i]); $sent++; $sentbytes += length($t); - if (lenth($t) > $sentmaxlength) { $sentmaxlength = length($t); } + if (length($t) > $sentmaxlength) { $sentmaxlength = length($t); } } # replace the prior buffer (if any) with the new data - $_[0]->pending =$t; + $pending[$i] =$t; } + } elsif ($pending[$i] ne "") { + # if there is no new information, send anything that's pending anyway. + # without this the last line of a log will never get sent. + # With this it waits one timeout, then gets sent. + + $filename = $thisfiletail->{"input"}; + $product = $$names[$i]; + sendlog($product, $pending[$i]); + $sent++; + $sentbytes += length($t); + if (length($t) > $sentmaxlength) { $sentmaxlength = length($t); } + $pending[$i] =''; } + } if (time > $nextreport) { sendlog("$host loglog-status sent lines=$sent bytes=$sentbytes maxlength=$sentmaxlength"); @@ -172,6 +197,7 @@ sub sendlog { ) or die "Socket could not be created : $!\n"; my $name = shift; my $msg = shift; + #print "sending log for $name $msg\n"; my $facility_i = $syslog_facilities{ $Facility } || 21; my $priority_i = $syslog_priorities{ $Priority } || 7; @@ -195,12 +221,15 @@ sub sendlog { if ($#messages = 1) { $message = "<$d>$ts $host loglog-${name}[$pid]: $messages[0]"; print $sock $message; + #print $message,"\n";; } else { for ($i=0; $i<$#message; $i++){ $message = "<$d>$ts $host loglog-${name}[$pid]: ($i/$#messages): $messages[$i]"; print $sock $message; + #print $message,"\n";; } } + @messages=(); $sock->close(); } @@ -244,13 +273,7 @@ sub interval { sub logit { my $object=shift; my @call=caller(1); - print # STDERR -# time()." ". - "\033[7m". - $call[3]." ".$object->{"input"}." ".join("",@_). - "\033[0m". - "\n" - if $object->debug; + print STDERR time()." ". "\033[7m". $call[3]." ".$object->{"input"}." ".join("",@_). "\033[0m". "\n" if $object->debug; } sub adjustafter { From 3900b75ec23026157726f32756eb97f3a1369ce4 Mon Sep 17 00:00:00 2001 From: David Lang Date: Mon, 5 Nov 2012 14:17:33 -0800 Subject: [PATCH 02/13] cleaned up handling of product name (conflicting use of the variable $name) --- loglog | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/loglog b/loglog index fba1522..fcde3c6 100755 --- a/loglog +++ b/loglog @@ -77,10 +77,12 @@ my %syslog_facilities = ( my @month = qw{Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec}; $SyslogHost = 'scribe'; +#$SyslogHost = 'localhost'; $Facility = 'local6'; $priority = 'info'; $pid = $$, $SyslogPort = 514, +#$SyslogPort = 10514, $maxloglength = 4000; $host = `hostname -s`; chomp($host); $conf = "/opt/disys/etc/loglog/$host.conf"; @@ -154,7 +156,7 @@ sendlog("status","finished init, configured to watch ".($#files+1)." logfiles, s # if there is a prior line send it if ($pending[$i] ne "") { $filename = $thisfiletail->{"input"}; - $product = $$names[$i]; + $product = $names[$i]; sendlog($product, $pending[$i]); $sent++; $sentbytes += length($t); @@ -169,7 +171,7 @@ sendlog("status","finished init, configured to watch ".($#files+1)." logfiles, s # With this it waits one timeout, then gets sent. $filename = $thisfiletail->{"input"}; - $product = $$names[$i]; + $product = $names[$i]; sendlog($product, $pending[$i]); $sent++; $sentbytes += length($t); @@ -195,9 +197,9 @@ sub sendlog { PeerPort => $SyslogPort, Proto => 'udp' ) or die "Socket could not be created : $!\n"; - my $name = shift; - my $msg = shift; - #print "sending log for $name $msg\n"; + my $product = shift; + my $msg = shift || "NO MESSAGE"; + #print "sending log for $product $msg\n"; my $facility_i = $syslog_facilities{ $Facility } || 21; my $priority_i = $syslog_priorities{ $Priority } || 7; @@ -219,12 +221,12 @@ sub sendlog { } push @messages, $msg; if ($#messages = 1) { - $message = "<$d>$ts $host loglog-${name}[$pid]: $messages[0]"; + $message = "<$d>$ts $host loglog-${product}[$pid]: $messages[0]"; print $sock $message; #print $message,"\n";; } else { for ($i=0; $i<$#message; $i++){ - $message = "<$d>$ts $host loglog-${name}[$pid]: ($i/$#messages): $messages[$i]"; + $message = "<$d>$ts $host loglog-${product}[$pid]: ($i/$#messages): $messages[$i]"; print $sock $message; #print $message,"\n";; } From 85ed72ded6c83643a20584505bbeb52fba46ba05 Mon Sep 17 00:00:00 2001 From: David Lang Date: Mon, 5 Nov 2012 15:13:15 -0800 Subject: [PATCH 03/13] added a check to avoid dieing if a file that we are told to monitor doesn't exist. --- loglog | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/loglog b/loglog index fcde3c6..c14c5fb 100755 --- a/loglog +++ b/loglog @@ -119,11 +119,13 @@ if (! -e $conf) { open (I, $conf); while () { - chomp; - ($filename,$product) = split(" ",$_); - if ($product eq '') {$product = "unknown";} - push(@files, File::Tail->new(name=>"$filename",debug=>$debug)); - push(@names, $product); + chomp; + ($filename,$product) = split(" ",$_); + if ($product eq '') {$product = "unknown";} + if (-e $filename) { + push(@files, File::Tail->new(name=>"$filename",debug=>$debug)); + push(@names, $product); + } } close I; From f775b8b6cad7328487b88c836be3008d2d9a3226 Mon Sep 17 00:00:00 2001 From: David Lang Date: Mon, 5 Nov 2012 15:17:57 -0800 Subject: [PATCH 04/13] fixed status line --- loglog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loglog b/loglog index c14c5fb..bb5784c 100755 --- a/loglog +++ b/loglog @@ -183,7 +183,7 @@ sendlog("status","finished init, configured to watch ".($#files+1)." logfiles, s } if (time > $nextreport) { - sendlog("$host loglog-status sent lines=$sent bytes=$sentbytes maxlength=$sentmaxlength"); + sendlog("status" "sent lines=$sent bytes=$sentbytes maxlength=$sentmaxlength"); $sent=0; $sentbytes=0; $sentmaxlength=0; $nextreport += $interval; } $mynewtimestamp = ctime(stat($loglogname)->mtime); From 8c0a3c286219aabfa2f88ef916e584175a7ceb27 Mon Sep 17 00:00:00 2001 From: David Lang Date: Mon, 5 Nov 2012 16:03:06 -0800 Subject: [PATCH 05/13] the recent changes dropped the filename from the log, this adds it back --- loglog | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/loglog b/loglog index bb5784c..ba3329b 100755 --- a/loglog +++ b/loglog @@ -146,6 +146,8 @@ sendlog("status","finished init, configured to watch ".($#files+1)." logfiles, s #print "$i $names[$i]\n"; #print " ".time."\n"; $thisfiletail = $files[$i]; + $filename = $thisfiletail->{"input"}; + $product = $names[$i]; if ($thisfiletail->predict == 0) { my $t = $thisfiletail->read; chomp($t); #print " $t\n"; @@ -157,9 +159,7 @@ sendlog("status","finished init, configured to watch ".($#files+1)." logfiles, s } else { # if there is a prior line send it if ($pending[$i] ne "") { - $filename = $thisfiletail->{"input"}; - $product = $names[$i]; - sendlog($product, $pending[$i]); + sendlog($product, $filename.': '.$pending[$i]); $sent++; $sentbytes += length($t); if (length($t) > $sentmaxlength) { $sentmaxlength = length($t); } @@ -172,9 +172,7 @@ sendlog("status","finished init, configured to watch ".($#files+1)." logfiles, s # without this the last line of a log will never get sent. # With this it waits one timeout, then gets sent. - $filename = $thisfiletail->{"input"}; - $product = $names[$i]; - sendlog($product, $pending[$i]); + sendlog($product, $filename.': '.$pending[$i]); $sent++; $sentbytes += length($t); if (length($t) > $sentmaxlength) { $sentmaxlength = length($t); } @@ -183,7 +181,7 @@ sendlog("status","finished init, configured to watch ".($#files+1)." logfiles, s } if (time > $nextreport) { - sendlog("status" "sent lines=$sent bytes=$sentbytes maxlength=$sentmaxlength"); + sendlog("status","sent lines=$sent bytes=$sentbytes maxlength=$sentmaxlength"); $sent=0; $sentbytes=0; $sentmaxlength=0; $nextreport += $interval; } $mynewtimestamp = ctime(stat($loglogname)->mtime); From 47ac343856d25cf9982f44f3e2e0027959643a2b Mon Sep 17 00:00:00 2001 From: David Lang Date: Wed, 7 Nov 2012 14:16:27 -0800 Subject: [PATCH 06/13] a better solution to the problem of files going away, File::Tail has a mode that will handle this gracefully and then find the file if it reappears. --- loglog | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/loglog b/loglog index ba3329b..66e617a 100755 --- a/loglog +++ b/loglog @@ -122,10 +122,8 @@ while () { chomp; ($filename,$product) = split(" ",$_); if ($product eq '') {$product = "unknown";} - if (-e $filename) { - push(@files, File::Tail->new(name=>"$filename",debug=>$debug)); - push(@names, $product); - } + push(@files, File::Tail->new(name=>"$filename",debug=>$debug,ignore_nonexistant=>TRUE)); + push(@names, $product); } close I; From e26b05c9df9c2d2973a9d6fb54fe88e317239f70 Mon Sep 17 00:00:00 2001 From: David Lang Date: Wed, 7 Nov 2012 14:24:55 -0800 Subject: [PATCH 07/13] add a check so that if a pending line is > 64K, send the data we have rather than continuing to wait and append more data to the line. Also, fixed maxlength report to report the size of the log sent, not the size of the line. --- loglog | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/loglog b/loglog index 66e617a..d6a5566 100755 --- a/loglog +++ b/loglog @@ -150,17 +150,18 @@ sendlog("status","finished init, configured to watch ".($#files+1)." logfiles, s my $t = $thisfiletail->read; chomp($t); #print " $t\n"; my $name = $thisfiletail->{"input"}; - if ($t =~ /^\s/ ) { + if ($t =~ /^\s/ || length($pending[$i]) > 65535) { # the line starts with whitespace, assume it's a continuation of the prior line # append it to the prior line, include a separator to indicate that we have done so + # if the line is longer than 64K, send what we have anyway. $pending[$i] .= " #015 ".$t; } else { # if there is a prior line send it if ($pending[$i] ne "") { sendlog($product, $filename.': '.$pending[$i]); $sent++; - $sentbytes += length($t); - if (length($t) > $sentmaxlength) { $sentmaxlength = length($t); } + $sentbytes += length($pending[$i]); + if (length($pending[$i]) > $sentmaxlength) { $sentmaxlength = length($pending[$i]); } } # replace the prior buffer (if any) with the new data $pending[$i] =$t; @@ -172,8 +173,8 @@ sendlog("status","finished init, configured to watch ".($#files+1)." logfiles, s sendlog($product, $filename.': '.$pending[$i]); $sent++; - $sentbytes += length($t); - if (length($t) > $sentmaxlength) { $sentmaxlength = length($t); } + $sentbytes += length($pending[$i]); + if (length($pending[$i]) > $sentmaxlength) { $sentmaxlength = length($pending[$i]); } $pending[$i] =''; } From 64b04934ff9050548ec6e011aba3530c5ac2a766 Mon Sep 17 00:00:00 2001 From: David Lang Date: Thu, 8 Nov 2012 17:01:32 -0800 Subject: [PATCH 08/13] added signal handler for USR1 that dumps all known variables to /var/tmp/loglog.dump --- loglog | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/loglog b/loglog index d6a5566..e3ba4d2 100755 --- a/loglog +++ b/loglog @@ -25,6 +25,92 @@ $debug=0; Proc::Daemon::Init; my $continue = 1; $SIG{TERM} = sub { $continue = 0 }; +$SIG{USR1} = sub { + open(DUMPFILE, ">/var/tmp/loglog.dump") or die "unable to write loglog dump file\n"; + print DUMPFILE "already_alive\n"; + print DUMPFILE Dumper(\$already_alive); + print DUMPFILE "conf\n"; + print DUMPFILE Dumper(\$conf); + print DUMPFILE "continue\n"; + print DUMPFILE Dumper(\$continue); + print DUMPFILE "d\n"; + print DUMPFILE Dumper(\$d); + print DUMPFILE "debug\n"; + print DUMPFILE Dumper(\$debug); + print DUMPFILE "Facility\n"; + print DUMPFILE Dumper(\$Facility); + print DUMPFILE "facility_i\n"; + print DUMPFILE Dumper(\$facility_i); + print DUMPFILE "filename\n"; + print DUMPFILE Dumper(\$filename); + print DUMPFILE "files\n"; + print DUMPFILE Dumper(\@files); + print DUMPFILE "host\n"; + print DUMPFILE Dumper(\$host); + print DUMPFILE "interval\n"; + print DUMPFILE Dumper(\$interval); + print DUMPFILE "loglogname\n"; + print DUMPFILE Dumper(\$loglogname); + print DUMPFILE "maxloglength\n"; + print DUMPFILE Dumper(\$maxloglength); + print DUMPFILE "message\n"; + print DUMPFILE Dumper(\$message); + print DUMPFILE "messagelength\n"; + print DUMPFILE Dumper(\$messagelength); + print DUMPFILE "messages\n"; + print DUMPFILE Dumper(\@messages); + print DUMPFILE "month\n"; + print DUMPFILE Dumper(\@month); + print DUMPFILE "msg\n"; + print DUMPFILE Dumper(\$msg); + print DUMPFILE "mynewtimestamp\n"; + print DUMPFILE Dumper(\$mynewtimestamp); + print DUMPFILE "mytimestamp\n"; + print DUMPFILE Dumper(\$mytimestamp); + print DUMPFILE "name\n"; + print DUMPFILE Dumper(\$name); + print DUMPFILE "names\n"; + print DUMPFILE Dumper(\@names); + print DUMPFILE "nfound\n"; + print DUMPFILE Dumper(\$nfound); + print DUMPFILE "pending\n"; + print DUMPFILE Dumper(\@pending); + print DUMPFILE "pid\n"; + print DUMPFILE Dumper(\$pid); + print DUMPFILE "priority\n"; + print DUMPFILE Dumper(\$priority); + print DUMPFILE "priority_i\n"; + print DUMPFILE Dumper(\$priority_i); + print DUMPFILE "product\n"; + print DUMPFILE Dumper(\$product); + print DUMPFILE "rin\n"; + print DUMPFILE Dumper(\$rin); + print DUMPFILE "sent\n"; + print DUMPFILE Dumper(\$sent); + print DUMPFILE "sentbytes\n"; + print DUMPFILE Dumper(\$sentbytes); + print DUMPFILE "sock\n"; + print DUMPFILE Dumper(\$sock); + print DUMPFILE "syslog_facilities\n"; + print DUMPFILE Dumper(\%syslog_facilities); + print DUMPFILE "SyslogHost\n"; + print DUMPFILE Dumper(\$SyslogHost); + print DUMPFILE "SyslogPort\n"; + print DUMPFILE Dumper(\$SyslogPort); + print DUMPFILE "syslog_priorities\n"; + print DUMPFILE Dumper(\%syslog_priorities); + print DUMPFILE "t\n"; + print DUMPFILE Dumper(\$t); + print DUMPFILE "thisfiletail\n"; + print DUMPFILE Dumper(\$thisfiletail); + print DUMPFILE "time\n"; + print DUMPFILE Dumper(\@time); + print DUMPFILE "timestamp\n"; + print DUMPFILE Dumper(\$timestamp); + print DUMPFILE "ts\n"; + print DUMPFILE Dumper(\$ts); + close(DUMPFILE); +}; my %syslog_priorities = ( emerg => 0, From 0b4dd8a682ae736b06b008eef49c5bb8f3aa2490 Mon Sep 17 00:00:00 2001 From: David Lang Date: Fri, 9 Nov 2012 16:20:33 -0800 Subject: [PATCH 09/13] added GetOptLong processing so that things that were hard-coded can now be set by the command line --- loglog | 43 ++++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/loglog b/loglog index e3ba4d2..895bf4a 100755 --- a/loglog +++ b/loglog @@ -6,12 +6,39 @@ use vars qw($VERSION); use IO::Socket; use Sys::Hostname; use Data::Dumper; +use Getopt::Long; $|=1; # Note that Proc::Daemon, and File::Tail are included inline. +$host = `hostname -s`; chomp($host); -# much of Net::Syslog has been copied. It's not used directly to allow logging with different programnames and to streamline dealing with long logs (re-coding send allows that routine to deal with long logs instead of needing the code in multiple places). - - +GetOptions("debug:i" => \$debug, + "file=s" => \$conf, + "help" => \$help, + "maxloglength=i" => \$maxloglength, + "port=s" => \$SyslogPort, + "priorty=s" => \$priority, + "facility=s" => \$Facility, + "server=s" => \$SyslogHost); + +if (!$debug) {$debug = 0;} +if (!$SyslogHost) {$SyslogHost = 'scribe';} +if (!$Facility) {$Facility = 'local6';} +if (!$priority) {$priority = 'info';} +if (!$SyslogPort) {$SyslogPort = 514,} +if (!$maxloglength) {$maxloglength = 4000;} +if (!$conf) {$conf = "/opt/disys/etc/loglog/$host.conf";} +if ($help) { + print STDERR "usage info\n"; + print STDERR " --help dump this info\n"; + print STDERR " --debug=#\n"; + print STDERR " --file=config file name\n"; + print STDERR " --maxloglength=#\n"; + print STDERR " --server=syslogserver\n"; + print STDERR " --port=syslogport\n"; + print STDERR " --priority=syslog priority\n"; + print STDERR " --facility=syslog facility\n"; + exit; +} $already_alive = `ps -ef | grep loglog | grep -v grep` + 0; if ($already_alive) { # print STDERR "Already running\n"; @@ -20,7 +47,6 @@ if ($already_alive) { $loglogname = File::Spec->rel2abs($0); $mytimestamp = ctime(stat($loglogname)->mtime); -$debug=0; Proc::Daemon::Init; my $continue = 1; @@ -162,16 +188,7 @@ my %syslog_facilities = ( ); my @month = qw{Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec}; -$SyslogHost = 'scribe'; -#$SyslogHost = 'localhost'; -$Facility = 'local6'; -$priority = 'info'; $pid = $$, -$SyslogPort = 514, -#$SyslogPort = 10514, -$maxloglength = 4000; -$host = `hostname -s`; chomp($host); -$conf = "/opt/disys/etc/loglog/$host.conf"; # print "config is $conf\n"; #these three arrays are tied together, element X of each array correspond to each other. From cb52924bdd3bcacae721a949a583bf8d10e87291 Mon Sep 17 00:00:00 2001 From: David Lang Date: Fri, 9 Nov 2012 17:05:33 -0800 Subject: [PATCH 10/13] created logpipe, a varient of loglog that reads from stdin and outputs to syslog. This is intended to read from a named pipe. --- logpipe | 272 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 272 insertions(+) create mode 100755 logpipe diff --git a/logpipe b/logpipe new file mode 100755 index 0000000..fcd5670 --- /dev/null +++ b/logpipe @@ -0,0 +1,272 @@ +#!/usr/bin/perl + +use File::stat; +use Time::localtime; +use vars qw($VERSION); +use IO::Socket; +use Sys::Hostname; +use Data::Dumper; +use Getopt::Long; +$|=1; +# Note that Proc::Daemon, and File::Tail are included inline. +$host = `hostname -s`; chomp($host); + +GetOptions("debug:i" => \$debug, + "tag=s" => \$product, + "help" => \$help, + "maxloglength=i" => \$maxloglength, + "port=s" => \$SyslogPort, + "priorty=s" => \$priority, + "facility=s" => \$Facility, + "server=s" => \$SyslogHost); + +if (!$debug) {$debug = 0;} +if (!$SyslogHost) {$SyslogHost = 'scribe';} +if (!$Facility) {$Facility = 'local6';} +if (!$priority) {$priority = 'info';} +if (!$SyslogPort) {$SyslogPort = 514,} +if (!$maxloglength) {$maxloglength = 4000;} +if (!$product) {$product = "unknown";} +if ($help) { + print STDERR "usage info\n"; + print STDERR " --help dump this info\n"; + print STDERR " --debug=#\n"; + print STDERR " --tag=tag for log message (loglog-)\n"; + print STDERR " --maxloglength=#\n"; + print STDERR " --server=syslogserver\n"; + print STDERR " --port=syslogport\n"; + print STDERR " --priority=syslog priority\n"; + print STDERR " --facility=syslog facility\n"; + exit; +} +$already_alive = `ps -ef | grep loglog | grep -v grep` + 0; +if ($already_alive) { + # print STDERR "Already running\n"; + exit; +} + +$loglogname = File::Spec->rel2abs($0); +$mytimestamp = ctime(stat($loglogname)->mtime); + +my $continue = 1; +$SIG{TERM} = sub { $continue = 0 }; +$SIG{USR1} = sub { + open(DUMPFILE, ">/var/tmp/loglog.dump") or die "unable to write loglog dump file\n"; + print DUMPFILE "already_alive\n"; + print DUMPFILE Dumper(\$already_alive); + print DUMPFILE "conf\n"; + print DUMPFILE Dumper(\$conf); + print DUMPFILE "continue\n"; + print DUMPFILE Dumper(\$continue); + print DUMPFILE "d\n"; + print DUMPFILE Dumper(\$d); + print DUMPFILE "debug\n"; + print DUMPFILE Dumper(\$debug); + print DUMPFILE "Facility\n"; + print DUMPFILE Dumper(\$Facility); + print DUMPFILE "facility_i\n"; + print DUMPFILE Dumper(\$facility_i); + print DUMPFILE "filename\n"; + print DUMPFILE Dumper(\$filename); + print DUMPFILE "files\n"; + print DUMPFILE Dumper(\@files); + print DUMPFILE "host\n"; + print DUMPFILE Dumper(\$host); + print DUMPFILE "interval\n"; + print DUMPFILE Dumper(\$interval); + print DUMPFILE "loglogname\n"; + print DUMPFILE Dumper(\$loglogname); + print DUMPFILE "maxloglength\n"; + print DUMPFILE Dumper(\$maxloglength); + print DUMPFILE "message\n"; + print DUMPFILE Dumper(\$message); + print DUMPFILE "messagelength\n"; + print DUMPFILE Dumper(\$messagelength); + print DUMPFILE "month\n"; + print DUMPFILE Dumper(\@month); + print DUMPFILE "msg\n"; + print DUMPFILE Dumper(\$msg); + print DUMPFILE "mynewtimestamp\n"; + print DUMPFILE Dumper(\$mynewtimestamp); + print DUMPFILE "mytimestamp\n"; + print DUMPFILE Dumper(\$mytimestamp); + print DUMPFILE "name\n"; + print DUMPFILE Dumper(\$name); + print DUMPFILE "names\n"; + print DUMPFILE Dumper(\$nfound); + print DUMPFILE "pending\n"; + print DUMPFILE Dumper(\$pending); + print DUMPFILE "pid\n"; + print DUMPFILE Dumper(\$pid); + print DUMPFILE "priority\n"; + print DUMPFILE Dumper(\$priority); + print DUMPFILE "priority_i\n"; + print DUMPFILE Dumper(\$priority_i); + print DUMPFILE "product\n"; + print DUMPFILE Dumper(\$product); + print DUMPFILE "rin\n"; + print DUMPFILE Dumper(\$rin); + print DUMPFILE "sent\n"; + print DUMPFILE Dumper(\$sent); + print DUMPFILE "sentbytes\n"; + print DUMPFILE Dumper(\$sentbytes); + print DUMPFILE "sock\n"; + print DUMPFILE Dumper(\$sock); + print DUMPFILE "syslog_facilities\n"; + print DUMPFILE Dumper(\%syslog_facilities); + print DUMPFILE "SyslogHost\n"; + print DUMPFILE Dumper(\$SyslogHost); + print DUMPFILE "SyslogPort\n"; + print DUMPFILE Dumper(\$SyslogPort); + print DUMPFILE "syslog_priorities\n"; + print DUMPFILE Dumper(\%syslog_priorities); + print DUMPFILE "t\n"; + print DUMPFILE Dumper(\$t); + print DUMPFILE "thisfiletail\n"; + print DUMPFILE Dumper(\@time); + print DUMPFILE "timestamp\n"; + print DUMPFILE Dumper(\$timestamp); + print DUMPFILE "ts\n"; + print DUMPFILE Dumper(\$ts); + close(DUMPFILE); +}; + +my %syslog_priorities = ( + emerg => 0, + emergency => 0, + alert => 1, + crit => 2, + critical => 2, + err => 3, + error => 3, + warning => 4, + notice => 5, + info => 6, + informational => 6, + debug => 7 +); + +my %syslog_facilities = ( + kern => 0, + kernel => 0, + user => 1, + mail => 2, + daemon => 3, + system => 3, + auth => 4, + syslog => 5, + internal => 5, + lpr => 6, + printer => 6, + news => 7, + uucp => 8, + cron => 9, + clock => 9, + authpriv => 10, + security2 => 10, + ftp => 11, + FTP => 11, + NTP => 11, + audit => 13, + alert => 14, + clock2 => 15, + local0 => 16, + local1 => 17, + local2 => 18, + local3 => 19, + local4 => 20, + local5 => 21, + local6 => 22, + local7 => 23, +); + +my @month = qw{Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec}; +$pid = $$, +# print "config is $conf\n"; + +#these three arrays are tied together, element X of each array correspond to each other. +$pending=''; + +my $timestamp = localtime(time); +sendlog("status","Startup (program timestamp is $mytimestamp)"); + +my $rin=''; +$interval = 5 * 60; # 5 minutes +$sent=0; $sentbytes=0; $sentmaxlength=0; $nextreport = time + $interval; +sendlog("status","finished init, configured to watch stdin, starting processing of logs"); + +while (<>) { + chomp; + $t=$_; + my $t = $thisfiletail->read; + #print " $t\n"; + if ($t =~ /^\s/ || length($pending) > 65535) { + # the line starts with whitespace, assume it's a continuation of the prior line + # append it to the prior line, include a separator to indicate that we have done so + # if the line is longer than 64K, send what we have anyway. + $pending .= " #015 ".$t; + } else { + # if there is a prior line send it + if ($pending ne "") { + sendlog($product, 'stdin: '.$pending); + $sent++; + $sentbytes += length($pending); + if (length($pending) > $sentmaxlength) { $sentmaxlength = length($pending); } + } + # replace the prior buffer (if any) with the new data + $pending =$t; + } +} +if (time > $nextreport) { + sendlog("status","sent lines=$sent bytes=$sentbytes maxlength=$sentmaxlength"); + $sent=0; $sentbytes=0; $sentmaxlength=0; $nextreport += $interval; +} +$mynewtimestamp = ctime(stat($loglogname)->mtime); +if ($mynewtimestamp ne $mytimestamp) { # Modified! die, and watchdog will restart + sendlog("status", "program updated, aborting"); + exit; +} + +sub sendlog { + my $sock = new IO::Socket::INET( + PeerAddr => $SyslogHost, + PeerPort => $SyslogPort, + Proto => 'udp' + ) or die "Socket could not be created : $!\n"; + my $product = shift; + my $msg = shift || "NO MESSAGE"; + #print "sending log for $product $msg\n"; + + my $facility_i = $syslog_facilities{ $Facility } || 21; + my $priority_i = $syslog_priorities{ $Priority } || 7; + + my $d = ( ( $facility_i << 3 ) | ($priority_i) ); + + my @time = CORE::localtime(time); + my $ts = + $month[ $time[4] ] . " " + . ( ( $time[3] < 10 ) ? ( " " . $time[3] ) : $time[3] ) . " " + . ( ( $time[2] < 10 ) ? ( "0" . $time[2] ) : $time[2] ) . ":" + . ( ( $time[1] < 10 ) ? ( "0" . $time[1] ) : $time[1] ) . ":" + . ( ( $time[0] < 10 ) ? ( "0" . $time[0] ) : $time[0] ); + + while (length($msg) > $maxloglength) { + $messagelength = rindex $msg, " ", $maxloglength ; #find the position of the last space before position $maxloglenth + push @messages, substr($msg, 0, $messagelength); + $msg = substr($msg, $messagelength); # skip the space + } + push @messages, $msg; + if ($#messages = 1) { + $message = "<$d>$ts $host loglog-${product}[$pid]: $messages[0]"; + print $sock $message; + #print $message,"\n";; + } else { + for ($i=0; $i<$#message; $i++){ + $message = "<$d>$ts $host loglog-${product}[$pid]: ($i/$#messages): $messages[$i]"; + print $sock $message; + #print $message,"\n";; + } + } + @messages=(); + $sock->close(); +} From 1504f477292dbb475fa99c4c44807075504e571d Mon Sep 17 00:00:00 2001 From: David Lang Date: Fri, 9 Nov 2012 19:59:15 -0800 Subject: [PATCH 11/13] fix hsndling of long messages. before the program would spin and eat all ram if it could not split a long message properly, now it has multiple fallbacks it trys to split on spaces, if there are no spaces it trys to split on tabs, if there are no tabs it trys to split on #015 (an escaped newline), and if that fails it splits on max log size --- loglog | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/loglog b/loglog index 895bf4a..69bbc27 100755 --- a/loglog +++ b/loglog @@ -30,18 +30,17 @@ if (!$conf) {$conf = "/opt/disys/etc/loglog/$host.conf";} if ($help) { print STDERR "usage info\n"; print STDERR " --help dump this info\n"; - print STDERR " --debug=#\n"; print STDERR " --file=config file name\n"; print STDERR " --maxloglength=#\n"; print STDERR " --server=syslogserver\n"; print STDERR " --port=syslogport\n"; print STDERR " --priority=syslog priority\n"; print STDERR " --facility=syslog facility\n"; + print STDERR " --debug=#\n"; exit; } $already_alive = `ps -ef | grep loglog | grep -v grep` + 0; if ($already_alive) { - # print STDERR "Already running\n"; exit; } @@ -188,8 +187,7 @@ my %syslog_facilities = ( ); my @month = qw{Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec}; -$pid = $$, -# print "config is $conf\n"; +$pid = $$; #these three arrays are tied together, element X of each array correspond to each other. @files=(); @@ -244,14 +242,11 @@ sendlog("status","finished init, configured to watch ".($#files+1)." logfiles, s } } for($i=0; $i<=$#files; $i++) { - #print "$i $names[$i]\n"; - #print " ".time."\n"; $thisfiletail = $files[$i]; $filename = $thisfiletail->{"input"}; $product = $names[$i]; if ($thisfiletail->predict == 0) { my $t = $thisfiletail->read; chomp($t); - #print " $t\n"; my $name = $thisfiletail->{"input"}; if ($t =~ /^\s/ || length($pending[$i]) > 65535) { # the line starts with whitespace, assume it's a continuation of the prior line @@ -301,7 +296,6 @@ sub sendlog { ) or die "Socket could not be created : $!\n"; my $product = shift; my $msg = shift || "NO MESSAGE"; - #print "sending log for $product $msg\n"; my $facility_i = $syslog_facilities{ $Facility } || 21; my $priority_i = $syslog_priorities{ $Priority } || 7; @@ -318,19 +312,30 @@ sub sendlog { while (length($msg) > $maxloglength) { $messagelength = rindex $msg, " ", $maxloglength ; #find the position of the last space before position $maxloglenth + $offset=1; + if ($messagelength == -1) { + $messagelength = rindex $msg, "\t", $maxloglength ; #find the position of the last tab before position $maxloglenth + $offset=1; + } + if ($messagelength == -1) { + $messagelength = rindex $msg, "#015", $maxloglength ; #find the position of the last escaped newline before position $maxloglenth + $offset=4; + } + if ($messagelength == -1) { + $messagelength = $maxloglength; #if all else fails, just trim the log at the max size, no matter where in the message it is. + $offset=0; + } push @messages, substr($msg, 0, $messagelength); - $msg = substr($msg, $messagelength); # skip the space + $msg = substr($msg, $messagelength +$offset); # skip the thing we are splitting on } push @messages, $msg; - if ($#messages = 1) { + if ($#messages == 0) { $message = "<$d>$ts $host loglog-${product}[$pid]: $messages[0]"; print $sock $message; - #print $message,"\n";; } else { - for ($i=0; $i<$#message; $i++){ - $message = "<$d>$ts $host loglog-${product}[$pid]: ($i/$#messages): $messages[$i]"; + for ($j=0; $j<=$#messages; $j++){ + $message = "<$d>$ts $host loglog-${product}[$pid]: (".($j+1).'/'.($#messages+1)."): $messages[$j]"; print $sock $message; - #print $message,"\n";; } } @messages=(); From ffe78f23686ccd354db583627deab001ea0cac6b Mon Sep 17 00:00:00 2001 From: David Lang Date: Fri, 9 Nov 2012 19:59:52 -0800 Subject: [PATCH 12/13] changed a lot of prior debug prints from being commented out to being controlled by the debug flag. The debug flag is a bitmask, so 1 enables one set of prints, 2 enables a different set, 3 enables both --- loglog | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/loglog b/loglog index 69bbc27..729557b 100755 --- a/loglog +++ b/loglog @@ -37,10 +37,16 @@ if ($help) { print STDERR " --priority=syslog priority\n"; print STDERR " --facility=syslog facility\n"; print STDERR " --debug=#\n"; + print STDERR " debug flags are a bitmask, turn on the bits for the debug features you want\n"; + print STDERR " 1 basic flow control\n"; + print STDERR " 2 sendlog splitting of log messages\n"; + print STDERR " 4 sendlog sending of log messages\n"; + print STDERR " 256 File::Tail internal debugging\n"; exit; } $already_alive = `ps -ef | grep loglog | grep -v grep` + 0; if ($already_alive) { + if ($debug & 1) { print STDERR "Already running\n"; } exit; } @@ -188,6 +194,7 @@ my %syslog_facilities = ( my @month = qw{Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec}; $pid = $$; +if ($debug & 1) { print "config is $conf\n"; } #these three arrays are tied together, element X of each array correspond to each other. @files=(); @@ -223,7 +230,7 @@ while () { chomp; ($filename,$product) = split(" ",$_); if ($product eq '') {$product = "unknown";} - push(@files, File::Tail->new(name=>"$filename",debug=>$debug,ignore_nonexistant=>TRUE)); + push(@files, File::Tail->new(name=>"$filename",debug=>($debug &256),ignore_nonexistant=>TRUE)); push(@names, $product); } close I; @@ -242,11 +249,14 @@ sendlog("status","finished init, configured to watch ".($#files+1)." logfiles, s } } for($i=0; $i<=$#files; $i++) { + if ($debug & 1 ){ print "$i $names[$i]\n"; } + if ($debug & 1 ){ print " ".time."\n"; } $thisfiletail = $files[$i]; $filename = $thisfiletail->{"input"}; $product = $names[$i]; if ($thisfiletail->predict == 0) { my $t = $thisfiletail->read; chomp($t); + if ($debug & 1 ){ print " $t\n";} my $name = $thisfiletail->{"input"}; if ($t =~ /^\s/ || length($pending[$i]) > 65535) { # the line starts with whitespace, assume it's a continuation of the prior line @@ -296,6 +306,7 @@ sub sendlog { ) or die "Socket could not be created : $!\n"; my $product = shift; my $msg = shift || "NO MESSAGE"; + if ($debug &1) {print "sending log for $product $msg\n";} my $facility_i = $syslog_facilities{ $Facility } || 21; my $priority_i = $syslog_priorities{ $Priority } || 7; @@ -314,29 +325,39 @@ sub sendlog { $messagelength = rindex $msg, " ", $maxloglength ; #find the position of the last space before position $maxloglenth $offset=1; if ($messagelength == -1) { + if ($debug &2) {print "trying splitting on tab\n";} $messagelength = rindex $msg, "\t", $maxloglength ; #find the position of the last tab before position $maxloglenth $offset=1; } if ($messagelength == -1) { + if ($debug &2) {print "trying splitting on #015\n";} $messagelength = rindex $msg, "#015", $maxloglength ; #find the position of the last escaped newline before position $maxloglenth $offset=4; } if ($messagelength == -1) { + if ($debug &2) {print "giving up splitting at $maxloglength\n";} $messagelength = $maxloglength; #if all else fails, just trim the log at the max size, no matter where in the message it is. $offset=0; } push @messages, substr($msg, 0, $messagelength); + if ($debug &2) {print "splitting message ",($messagelength+1)," offset $offset, ",($#messages+1)," part, last part: $messages[-1]\n";} $msg = substr($msg, $messagelength +$offset); # skip the thing we are splitting on } + if ($debug &2) {print "message $msg\n";} push @messages, $msg; + if ($debug & 4) {print $#messages, " parts in this message\n";} if ($#messages == 0) { $message = "<$d>$ts $host loglog-${product}[$pid]: $messages[0]"; + if ($debug &4) {print "sending single",$message,"\n";} print $sock $message; } else { + if ($debug &4) {print "multipart message $j $#messages\n";} for ($j=0; $j<=$#messages; $j++){ $message = "<$d>$ts $host loglog-${product}[$pid]: (".($j+1).'/'.($#messages+1)."): $messages[$j]"; + if ($debug &4) {print "sending multipart $j ",$message,"\n";} print $sock $message; } + if ($debug &4) {print "finished multipart message\n";} } @messages=(); $sock->close(); From 9e8cb68b85f655be56322b47d0643071ed2dfac5 Mon Sep 17 00:00:00 2001 From: David Lang Date: Wed, 14 Nov 2012 11:33:07 -0800 Subject: [PATCH 13/13] modified sendlog to take hostname field and modified config file to have a third column holding the hostname so that logs can be gathered on one server, but then sent out as if they were being sent from the system that created the log --- loglog | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/loglog b/loglog index 729557b..f56c76d 100755 --- a/loglog +++ b/loglog @@ -200,26 +200,27 @@ if ($debug & 1) { print "config is $conf\n"; } @files=(); @names=(); @pending=(); +@hostnames=(); my $timestamp = localtime(time); -sendlog("status","Startup (program timestamp is $mytimestamp)"); +sendlog("status",$host,"Startup (program timestamp is $mytimestamp)"); if (! -e $conf) { - sendlog("$host loglog $timestamp Creating default logfile list"); + sendlog("status",$host,"Creating default logfile list"); open (C, ">$conf"); open (I, "find -L /opt/jboss -type f -mtime -1 -name '*.log' -print | grep -v 2012 |"); while () { - print C "$_ unknown"; + print C "$_ unknown $host"; } close I; open (I, "find -L /opt/mule -type f -mtime -1 -name '*.log' -print | grep -v 2012 |"); while () { - print C "$_ unknown"; + print C "$_ unknown $host"; } close I; open (I, "find -L /opt/di/system/jboss -type f -mtime -1 -name '*.log' -print | grep -v 2012 |"); while () { - print C "$_ unknown"; + print C "$_ unknown $host"; } close I; close C; @@ -228,17 +229,19 @@ if (! -e $conf) { open (I, $conf); while () { chomp; - ($filename,$product) = split(" ",$_); + ($filename,$product,$sendhost) = split(" ",$_); if ($product eq '') {$product = "unknown";} + if ($sendhost eq '') {$product = $host;} push(@files, File::Tail->new(name=>"$filename",debug=>($debug &256),ignore_nonexistant=>TRUE)); push(@names, $product); + push(@hostnames, $sendhost); } close I; my $rin=''; $interval = 5 * 60; # 5 minutes $sent=0; $sentbytes=0; $sentmaxlength=0; $nextreport = time + $interval; -sendlog("status","finished init, configured to watch ".($#files+1)." logfiles, starting processing of logs"); +sendlog("status",$host,"finished init, configured to watch ".($#files+1)." logfiles, starting processing of logs"); while ($continue) { $nfound=File::Tail::select(undef,undef,undef,60,@files); @@ -266,7 +269,7 @@ sendlog("status","finished init, configured to watch ".($#files+1)." logfiles, s } else { # if there is a prior line send it if ($pending[$i] ne "") { - sendlog($product, $filename.': '.$pending[$i]); + sendlog($product,$hostnames[$i], $filename.': '.$pending[$i]); $sent++; $sentbytes += length($pending[$i]); if (length($pending[$i]) > $sentmaxlength) { $sentmaxlength = length($pending[$i]); } @@ -279,7 +282,7 @@ sendlog("status","finished init, configured to watch ".($#files+1)." logfiles, s # without this the last line of a log will never get sent. # With this it waits one timeout, then gets sent. - sendlog($product, $filename.': '.$pending[$i]); + sendlog($product,$hostnames[$i], $filename.': '.$pending[$i]); $sent++; $sentbytes += length($pending[$i]); if (length($pending[$i]) > $sentmaxlength) { $sentmaxlength = length($pending[$i]); } @@ -288,12 +291,12 @@ sendlog("status","finished init, configured to watch ".($#files+1)." logfiles, s } if (time > $nextreport) { - sendlog("status","sent lines=$sent bytes=$sentbytes maxlength=$sentmaxlength"); + sendlog("status",$host,"sent lines=$sent bytes=$sentbytes maxlength=$sentmaxlength"); $sent=0; $sentbytes=0; $sentmaxlength=0; $nextreport += $interval; } $mynewtimestamp = ctime(stat($loglogname)->mtime); if ($mynewtimestamp ne $mytimestamp) { # Modified! die, and watchdog will restart - sendlog("status", "program updated, aborting"); + sendlog("status", $host,"program updated, aborting"); exit; } } @@ -305,6 +308,7 @@ sub sendlog { Proto => 'udp' ) or die "Socket could not be created : $!\n"; my $product = shift; + my $sendhost = shift; my $msg = shift || "NO MESSAGE"; if ($debug &1) {print "sending log for $product $msg\n";} @@ -347,13 +351,13 @@ sub sendlog { push @messages, $msg; if ($debug & 4) {print $#messages, " parts in this message\n";} if ($#messages == 0) { - $message = "<$d>$ts $host loglog-${product}[$pid]: $messages[0]"; + $message = "<$d>$ts $sendhost loglog-${product}[$pid]: $messages[0]"; if ($debug &4) {print "sending single",$message,"\n";} print $sock $message; } else { if ($debug &4) {print "multipart message $j $#messages\n";} for ($j=0; $j<=$#messages; $j++){ - $message = "<$d>$ts $host loglog-${product}[$pid]: (".($j+1).'/'.($#messages+1)."): $messages[$j]"; + $message = "<$d>$ts $sendhost loglog-${product}[$pid]: (".($j+1).'/'.($#messages+1)."): $messages[$j]"; if ($debug &4) {print "sending multipart $j ",$message,"\n";} print $sock $message; }