############################################ ## ## ## WebLog ## ## by Darryl Burgdorf ## ## (e-mail burgdorf@awsd.com) ## ## ## ## version: 2.20 ## ## last modified: 11/8/99 ## ## copyright (c) 1999 ## ## ## ## latest version is available from ## ## http://awsd.com/scripts/ ## ## ## ############################################ # COPYRIGHT NOTICE: # # Copyright 1999 Darryl C. Burgdorf. All Rights Reserved. # # This program may be used and modified free of charge by anyone, so # long as this copyright notice and the header above remain intact. # Selling the code for this program without prior written consent is # expressly forbidden. Obtain permission before redistributing this # program over the Internet or in any other medium. In all cases # copyright and header must remain intact. # # This program is distributed "as is" and without warranty of any # kind, either express or implied. (Some states do not allow the # limitation or exclusion of liability for incidental or consequential # damages, so this notice may not apply to you.) In no event shall # the liability of Darryl C. Burgdorf and/or Affordable Web Space # Design for any damages, losses and/or causes of action exceed the # total amount paid by the user for this software. # VERSION HISTORY: # # 2.20 11/08/99 Added support for MS extended log format # Added errors to "details" report # "Trimmed" IPs/domains to improve visitor count # (This also makes the IP database smaller!) # Added agent/platform info to details report # "Singularized" all agent/platform list entries # Fixed recognition of newer versions of Opera # Split robots and harvesters in platform list # Trapped for obviously corrupted agent names # Updated and greatly expanded keywords parsing # Squashed bug in "Top Referers" calculation # Improved EOM File creation criteria # Added $TotalReset flag # 2.12 05/13/99 Corrected minor bug in $DefaultPageName def # "Single-Quoted" several variable definitions # Allowed for possibility of spaces in user IDs # Added "Non-Standard Domain" to top domains list # Added $bodyspec, $headerfile and $footerfile # Fixed bug preventing re-gzipping of files # 2.11 04/20/99 Put IP resolution back into main processing loop # 2.10 03/30/99 Replaced resolved IPs list with DBM file # Removed IP resolution from main processing loop # Eliminated copying of @Accesses array # Improved "memory management" of disabled options # Added stripping of "/../" from file names # Expanded "Verbose" notifications # Took care of a few minor "format bug" fixes # Added "E-Mail Harvesters" to agents list # Corrected counting of some AOL as MSIE # Corrected "Netscape" to "Netscape Navigator" # Improved parsing of "ambiguous" Win platforms # Added MSN to search engines in keywords list # Removed final spaces from keywords lists # Allowed *all* ref domains to be "lower-cased" # Various other minor tweaks # 2.03 07/04/98 Added "top referers" list # Added Windows 98 to platform list # Allowed for better EOM processing # Fixed handling of "mixed" type log files # Added date to details report listings # Improved second-level parsing of non-US domains # Allowed for NT-style "\" file path delimiters # 2.02 05/02/98 Trapped "run on" URLs # Added $HourOffset configuration variable # Added "$Verbose" flag # 2.01 04/06/98 Fixed path bug in "wildcarded" file names # Fixed bug in unzipping of multiple zipped files # Fixed bug preventing print of "null" reports # Fixed bug in hourly visit & page view counting # Corrected multi-file reset of %agentcounter # Added "Snap!" to keywords breakdown # Separated "Excite" and "Netfind" in breakdown # 2.00 03/09/98 Incorporated referer & agent logging # Added "search keywords" logging # Added resolution of IP numbers # Added ability to "wildcard" log file name # Added "page views" logging to main report # Allowed user selection of info to be graphed # Fixed bug causing graphs to flake with large logs # Simplified configuration of file locations # Eliminated need to "pre-create" report files # Eliminated separate "country codes" file # Allowed "autodetection" of log file type # Set $NoSessions to disable *all* sessions logging # Expanded list to include all HTTP/1.1 status codes # Script now re-compresses gzip'd log files # Made the usual collection of minor format tweaks # 1.12 01/01/98 Added tracking of user IDs and associated domains # Made complete file listing optional # Cleaned up minor glitch in EOM listings # Adjusted regex to allow single-digit log file dates # 1.11 08/06/97 Fixed minor bug in $EndDate calculation # 1.10 08/04/97 Finally eliminated "one day's log per day" limit # Added "log only new" option # Improved handling of various server error codes # Removed unnecessary "summary" report # Fixed minor bug in session counting # 1.05 05/24/97 Added comma delineation to all numbers # Added "record book" report # Fixed new bug in "top" domain logging # 1.04 05/03/97 Changed month/day logs from domains to sessions # Added sessions to day-of-week log # Allowed disabling of above sessions logging # Allowed "details" log of only one day's accesses # Enabled "top" domain logging even if domains off # Added ability to log "second level" domains # Fixed display bugs if no accesses on first day # The typical selection of minor bug fixes # 1.03 02/02/97 Added support for combined log files # Added option to include file query information # Corrected handling of zipped files # Corrected bug in EOM report wrapping (Dec/Jan) # 1.02 12/11/96 Added "running total" of user sessions # 1.01 12/08/96 Added "user sessions" to details report # Corrected day counting in details report # 1.00 12/02/96 Initial "public" release sub MainProg { if ($Verbose) { print "Log Analysis for $SystemName\n"; } &Initialize; if ($LogFile =~ /\*/) { if ($LogFile =~ /(.*)[\/\\]([^\/\\]+)/) { $dir = $1; $LogFile = $2; } else { $dir = ""; } $LogFile =~ s/(\W)/\\$1/g; $LogFile =~ s/\\\*/\.\*/g; opendir (DIR,$dir) || die " Error: Unable to read log files directory\n"; @files = readdir (DIR); closedir (DIR); foreach $file (@files) { if ($file =~ $LogFile) { if ($dir) { $fullfile = $dir."/".$file; } else { $fullfile = $file; } push (@LogFiles,$fullfile); } } } else { push (@LogFiles,$LogFile); } foreach $LogFile (sort @LogFiles) { $LogFile =~ s/.gz$//; if (-e "$LogFile.gz") { $ZipLock = 1; system ("gunzip -q $LogFile"); } $OldFile = $ReportFile; $time = time; $RealDate = (localtime($time+($HourOffset*3600)))[3]; $time = (stat("$LogFile"))[9]; ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime($time+($HourOffset*3600)); $year += 1900; if ($EOMFile && (($mday==1) || ($RealDate < $mday))) { $ReportFile = $EOMFile; if ($mday == 1) { ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime($time-86400+($HourOffset*3600)); $year += 1900; } $sec = 59; $min = 59; $hour = 23; $eom = $mon+1; if (length($eom) == 1) { $eom = "0".$eom; } $EOMDate="$year $eom $mday 23 59 59"; &ResetVars; &ReadOldFile; &ReadLog; if ($LogLineCount || ($EndDate gt $InitialEndDate)) { unless ($NoSessions) { &GetSessions; } if ($Verbose) { print " Generating EOM Report\n"; } &PrintReport; } $ReportFile = $OldFile; $EOMDate = ""; ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime($time+($HourOffset*3600)); $year += 1900; } &ResetVars; &ReadOldFile; &ReadLog; if ($LogLineCount || ($EndDate gt $InitialEndDate)) { if ($AgentsFile) { &PrintAgentsReport; } if ($RefsFile) { &PrintRefsReport; } if ($KeywordsFile) { &PrintKeywordsReport; } unless ($NoSessions) { &GetSessions; } if ($Verbose) { print " Generating Main Report\n"; } &PrintReport; if ($DetailsFile) { &PrintHostDetailsReport; } } if ($ZipLock) { system ("gzip $LogFile"); $ZipLock = 0; } } if ($Verbose) { print "Report Complete\n"; } } sub Initialize { $version = "2.20"; @months=(Jan,Feb,Mar,Apr,May,Jun,Jul,Aug,Sep,Oct,Nov,Dec); %MonthToNumber= ('Jan','01','Feb','02','Mar','03', 'Apr','04','May','05','Jun','06', 'Jul','07','Aug','08','Sep','09', 'Oct','10','Nov','11','Dec','12'); %NumberToMonth= ('01','Jan','02','Feb','03','Mar', '04','Apr','05','May','06','Jun', '07','Jul','08','Aug','09','Sep', '10','Oct','11','Nov','12','Dec'); %DoWCounter= (1,'Sun',2,'Mon',3,'Tue',4,'Wed',5,'Thu',6,'Fri',7,'Sat'); $DefaultPageName = '(index\.(((s|p)*html*|cgi))|welcome\.(((s|p)*html*|cgi)))'; if ($NoSessions) { $DetailsFile = ""; if ($GraphBase eq "visits") { $GraphBase = ""; } } unless ($RefsFile) { $KeywordsFile = ""; } if ($NoResolve) { $IPLog = ""; } %RespCodes = ( '100','Code 100 Continue', '101','Code 101 Switching Protocols', '200','Code 200 OK', '201','Code 201 Created', '202','Code 202 Accepted', '203','Code 203 Non-Authoritative Information', '204','Code 204 No Content', '205','Code 205 Reset Content', '206','Code 206 Partial Content', '300','Code 300 Multiple Choices', '301','Code 301 Moved Permanently', '302','Code 302 Moved Temporarily', '303','Code 303 See Other', '304','Code 304 Not Modified', '305','Code 305 Use Proxy', '400','Code 400 Bad Request', '401','Code 401 Unauthorized', '402','Code 402 Payment Required', '403','Code 403 Forbidden', '404','Code 404 Not Found', '405','Code 405 Method Not Allowed', '406','Code 406 Not Acceptable', '407','Code 407 Proxy Authentication Required', '408','Code 408 Request Time-Out', '409','Code 409 Conflict', '410','Code 410 Gone', '411','Code 411 Length Required', '412','Code 412 Precondition Failed', '413','Code 413 Request Entity Too Large', '414','Code 414 Request-URI Too Large', '415','Code 415 Unsupported Media Type', '500','Code 500 Internal Server Error', '501','Code 501 Not Implemented', '502','Code 502 Bad Gateway', '503','Code 503 Service Unavailable', '504','Code 504 Gateway Time-Out', '505','Code 505 HTTP Version Not Supported' ); %CountryCodes = ( 'ad','Andorra', 'ae','United Arab Emirates', 'af','Afghanistan', 'ag','Antigua and Barbuda', 'ai','Anguilla', 'al','Albania', 'am','Armenia', 'an','Netherlands Antilles', 'ao','Angola', 'aq','Antarctica', 'ar','Argentina', 'as','American Samoa', 'at','Austria', 'au','Australia', 'aw','Aruba', 'az','Azerbaijan', 'ba','Bosnia and Herzegovina', 'bb','Barbados', 'bd','Bangladesh', 'be','Belgium', 'bf','Burkina Faso', 'bg','Bulgaria', 'bh','Bahrain', 'bi','Burundi', 'bj','Benin', 'bm','Bermuda', 'bn','Brunei Darussalam', 'bo','Bolivia', 'br','Brazil', 'bs','Bahamas', 'bt','Bhutan', 'bv','Bouvet Island', 'bw','Botswana', 'by','Belarus', 'bz','Belize', 'ca','Canada', 'cc','Cocos (Keeling) Islands', 'cf','Central African Republic', 'cg','Congo', 'ch','Switzerland', 'ci','Cote DIvoire (Ivory Coast)', 'ck','Cook Islands', 'cl','Chile', 'cm','Cameroon', 'cn','China', 'co','Colombia', 'cr','Costa Rica', 'cs','Czechoslovakia (Former)', 'cu','Cuba', 'cv','Cape Verde', 'cx','Christmas Island', 'cy','Cyprus', 'cz','Czech Republic', 'de','Germany', 'dj','Djibouti', 'dk','Denmark', 'dm','Dominica', 'do','Dominican Republic', 'dz','Algeria', 'ec','Ecuador', 'ee','Estonia', 'eg','Egypt', 'eh','Western Sahara', 'er','Eritrea', 'es','Spain', 'et','Ethiopia', 'fi','Finland', 'fj','Fiji', 'fk','Falkland Islands (Malvinas)', 'fm','Micronesia', 'fo','Faroe Islands', 'fr','France', 'fx','France (Metropolitan)', 'ga','Gabon', 'gb','Great Britain (UK)', 'gd','Grenada', 'ge','Georgia', 'gf','French Guiana', 'gh','Ghana', 'gi','Gibraltar', 'gl','Greenland', 'gm','Gambia', 'gn','Guinea', 'gp','Guadeloupe', 'gq','Equatorial Guinea', 'gr','Greece', 'gs','S. Georgia and S. Sandwich Islands', 'gt','Guatemala', 'gu','Guam', 'gw','Guinea-Bissau', 'gy','Guyana', 'hk','Hong Kong', 'hm','Heard and McDonald Islands', 'hn','Honduras', 'hr','Croatia (Hrvatska)', 'ht','Haiti', 'hu','Hungary', 'id','Indonesia', 'ie','Ireland', 'il','Israel', 'in','India', 'io','British Indian Ocean Territory', 'iq','Iraq', 'ir','Iran', 'is','Iceland', 'it','Italy', 'jm','Jamaica', 'jo','Jordan', 'jp','Japan', 'ke','Kenya', 'kg','Kyrgyzstan', 'kh','Cambodia', 'ki','Kiribati', 'km','Comoros', 'kn','Saint Kitts and Nevis', 'kp','North Korea', 'kr','South Korea', 'kw','Kuwait', 'ky','Cayman Islands', 'kz','Kazakhstan', 'la','Laos', 'lb','Lebanon', 'lc','Saint Lucia', 'li','Liechtenstein', 'lk','Sri Lanka', 'lr','Liberia', 'ls','Lesotho', 'lt','Lithuania', 'lu','Luxembourg', 'lv','Latvia', 'ly','Libya', 'ma','Morocco', 'mc','Monaco', 'md','Moldova', 'mg','Madagascar', 'mh','Marshall Islands', 'mk','Macedonia', 'ml','Mali', 'mm','Myanmar', 'mn','Mongolia', 'mo','Macau', 'mp','Northern Mariana Islands', 'mq','Martinique', 'mr','Mauritania', 'ms','Montserrat', 'mt','Malta', 'mu','Mauritius', 'mv','Maldives', 'mw','Malawi', 'mx','Mexico', 'my','Malaysia', 'mz','Mozambique', 'na','Namibia', 'nc','New Caledonia', 'ne','Niger', 'nf','Norfolk Island', 'ng','Nigeria', 'ni','Nicaragua', 'nl','Netherlands', 'no','Norway', 'np','Nepal', 'nr','Nauru', 'nt','Neutral Zone', 'nu','Niue', 'nz','New Zealand (Aotearoa)', 'om','Oman', 'pa','Panama', 'pe','Peru', 'pf','French Polynesia', 'pg','Papua New Guinea', 'ph','Philippines', 'pk','Pakistan', 'pl','Poland', 'pm','St. Pierre and Miquelon', 'pn','Pitcairn', 'pr','Puerto Rico', 'pt','Portugal', 'pw','Palau', 'py','Paraguay', 'qa','Qatar', 're','Reunion', 'ro','Romania', 'ru','Russian Federation', 'rw','Rwanda', 'sa','Saudi Arabia', 'sb','Solomon Islands', 'sc','Seychelles', 'sd','Sudan', 'se','Sweden', 'sg','Singapore', 'sh','St. Helena', 'si','Slovenia', 'sj','Svalbard and Jan Mayen Islands', 'sk','Slovak Republic', 'sl','Sierra Leone', 'sm','San Marino', 'sn','Senegal', 'so','Somalia', 'sr','Suriname', 'st','Sao Tome and Principe', 'su','USSR (Former)', 'sv','El Salvador', 'sy','Syria', 'sz','Swaziland', 'tc','Turks and Caicos Islands', 'td','Chad', 'tf','French Southern Territories', 'tg','Togo', 'th','Thailand', 'tj','Tajikistan', 'tk','Tokelau', 'tm','Turkmenistan', 'tn','Tunisia', 'to','Tonga', 'tp','East Timor', 'tr','Turkey', 'tt','Trinidad and Tobago', 'tv','Tuvalu', 'tw','Taiwan', 'tz','Tanzania', 'ua','Ukraine', 'ug','Uganda', 'uk','United Kingdom', 'um','US Minor Outlying Islands', 'us','United States', 'uy','Uruguay', 'uz','Uzbekistan', 'va','Vatican City State (Holy See)', 'vc','Saint Vincent and the Grenadines', 've','Venezuela', 'vg','Virgin Islands (British)', 'vi','Virgin Islands (US)', 'vn','Viet Nam', 'vu','Vanuatu', 'wf','Wallis and Futuna Islands', 'ws','Samoa', 'ye','Yemen', 'yt','Mayotte', 'yu','Yugoslavia', 'za','South Africa', 'zm','Zambia', 'zr','Zaire', 'zw','Zimbabwe', 'com','US Commercial', 'edu','US Educational', 'gov','US Government', 'int','International', 'mil','US Military', 'net','Network', 'org','Non-Profit Organization', 'arpa','Old-Style Arpanet', 'nato','NATO Field', 'xxx','Unresolved Domain', 'ooo','Non-Standard Domain' ); } sub ResetVars { &date_to_count(($mon+1),$mday,($year-1900)); $DayBreak = $perp_days; $CurrDate="$mday $months[$mon] $year"; $CurrMonth="$months[$mon]"; $CurrYear="$year"; $InitialEndDate="0000 00 00 00 00 00"; $EndDate="0000 00 00 00 00 00"; $FileEndDate="0000 00 00 00 00 00"; $FileStartDate="9999 99 99 99 99 99"; $AveragePages = 0; $DailyTotalBytesCounter = 0; $DailyTotalHitsCounter = 0; $DailyTotalPViewsCounter = 0; $DailyTotalVisitsCounter = 0; $DoWTotalBytesCounter = 0; $DoWTotalHitsCounter = 0; $DoWTotalPViewsCounter = 0; $DoWTotalVisitsCounter = 0; $GoDaily = 0; $GoMonthly = 0; $HourlyTotalBytesCounter = 0; $HourlyTotalHitsCounter = 0; $HourlyTotalPViewsCounter = 0; $HourlyTotalVisitsCounter = 0; $LocalPercent = 0; $MonthlyTotalBytesCounter = 0; $MonthlyTotalHitsCounter = 0; $MonthlyTotalPViewsCounter = 0; $MonthlyTotalVisitsCounter = 0; $OutsidePercent = 0; $TodayBytes = 0; $TodayErrors = 0; $TodayHits = 0; $TodayHosts = 0; $TodayKB = 0; $TodayLocal = 0; $TodayOutside = 0; $TodayPages = 0; $TopDomain = ""; $endhour = 0; $logsegment = ""; $refscounter = 0; $sessionstime = 0; $usersessions = 0; %BytesDay = (); %BytesFileCounter = (); %BytesHour = (); %BytesUserIDCounter = (); %DayFilesCounter = (); %DoWBytesCounter = (); %DoWFilesCounter = (); %DoWPViewsCounter = (); %DoWSessionsCounter = (); %DomainsBytesCounter = (); %DomainsDay = (); %DomainsFilesCounter = (); %HitsFileCounter = (); %HitsUserIDCounter = (); %HourFilesCounter = (); %LastAccessDomain = (); %LastAccessFile = (); %LastAccessUserID = (); %MonthlyBytesCounter = (); %MonthlyCounter = (); %MonthlyFilesCounter = (); %MonthlyPViewsCounter = (); %MonthlySessionsCounter = (); %PViewsDay = (); %PViewsHour = (); %Resolved = (); %TargetCounter = (); %TodayDomains = (); %TodayHosts = (); %TodayPages = (); %TopDomainAccess = (); %TopDomainBytesCounter = (); %TopDomainFilesCounter = (); %VisitsHour = (); %agentcounter = (); %agentreport = (); %altavista = (); %askjeeves = (); %cnet = (); %combinedreport = (); %dates = (); %day_counts = (); %dayusersessions = (); %dogpile = (); %dowusersessions = (); %excite = (); %euroferret = (); %euroseek = (); %fnfBytesFileCounter = (); %fnfHitsFileCounter = (); %fnfLastAccessFile = (); %google = (); %goto = (); %hostlist = (); %hotbot = (); %hourusersessions = (); %infoseek = (); %looksmart = (); %lycos = (); %magellan = (); %mamma = (); %metacrawler = (); %monthusersessions = (); %msn = (); %netfind = (); %netscape = (); %northernlight = (); %othersearch = (); %planetsearch = (); %platformreport = (); %prevday = (); %prevvisit = (); %refdomain = (); %savvysearch = (); %snap = (); %webcrawler = (); %yahoo = (); @Accesses = (); @Sessions = (); @files = (); @hosts = (); @lines = (); } sub ReadOldFile { if ($Verbose) { print " Reading Old Report File: $OldFile\n"; } $endTag=""; $InSegment = 0; return unless (-e "$FileDir/$OldFile"); open (OLD,"$FileDir/$OldFile") || die " Error Opening File: $OldFile\n"; while () { chop; last if (m##oi); unless ($endhour || $logsegment) { ($endhour,$endminute,$endsec, $endday,$endmonth,$endyear) = m#^.*(\d\d):(\d\d):(\d\d) (\d\d) (\w\w\w)\w* (\d\d\d\d)#o; if ($endhour) { $EndDate= "$endyear $MonthToNumber{$endmonth} $endday $endhour $endminute $endsec"; $InitialEndDate = $EndDate; } } if (//oi) { $logsegment=$1; $InSegment=1; if ($logsegment ne 'records') { ; ; ; } next; } $InSegment = 0 if ($_ eq $endTag); next if (!$InSegment); if ($logsegment eq "monthly") { ($Accesses,$Bytes,$Domains,$PViews,$Month) = m#^\s*([\d,]+)\s+([\d,]+)\s+([\d,]+|-)\s+([\d,]+)\s+(\w.{7})#o; $Accesses =~ s/,//g; $Bytes =~ s/,//g; $Domains =~ s/,//g; $PViews =~ s/,//g; $MonthlyFilesCounter{$Month}=$Accesses; $MonthlyBytesCounter{$Month}=$Bytes; $MonthlySessionsCounter{$Month}=$Domains; $MonthlyPViewsCounter{$Month}=$PViews; $MonthlyTotalBytesCounter+=$Bytes; $MonthlyTotalHitsCounter+=$Accesses; $MonthlyTotalVisitsCounter+=$Domains; $MonthlyTotalPViewsCounter+=$PViews; next; } elsif ($logsegment eq "daily") { ($Accesses,$Bytes,$Domains,$PViews,$Day,$Month) = m#^\s*([\d,]+)\s+([\d,]+)\s+([\d,]+|-)\s+([\d,]+)\s+(\d\d)\s+(\w\w\w)#o; if ($Month) { $Accesses =~ s/,//g; $Bytes =~ s/,//g; $Domains =~ s/,//g; $PViews =~ s/,//g; $Year=$CurrYear; if (($Month =~ "(Nov|Dec)") && ($CurrMonth =~ "(Jan|Feb)")) { $Year=$Year-1; } $Today="$Year $MonthToNumber{$Month} $Day"; $DayFilesCounter{$Today}=$Accesses; $BytesDay{$Today}=$Bytes; $DomainsDay{$Today}=$Domains; $PViewsDay{$Today}=$PViews; &date_to_count(int($MonthToNumber{$Month}),$Day,($Year-1900)); if ($DayBreak-$perp_days < 35) { $DailyTotalBytesCounter+=$Bytes; $DailyTotalHitsCounter+=$Accesses; $DailyTotalVisitsCounter+=$Domains; $DailyTotalPViewsCounter+=$PViews; } $Today="$Today 00 00 00"; $EndDate=$Today if ($Today gt $EndDate); } next; } elsif ($logsegment eq "dayofweek") { next if ($TotalReset); ($Accesses,$Bytes,$Sessions,$PViews,$Day) = m#^\s*([\d,]+)\s+([\d,]+)\s+([\d,]+|-)\s+([\d,]+)\s+(\w\w\w)#o; $Accesses =~ s/,//g; $Bytes =~ s/,//g; $Sessions =~ s/,//g; $PViews =~ s/,//g; $DoWFilesCounter{$Day}=$Accesses; $DoWBytesCounter{$Day}=$Bytes; $DoWSessionsCounter{$Day}=$Sessions; $DoWPViewsCounter{$Day}=$PViews; $DoWTotalBytesCounter+=$Bytes; $DoWTotalHitsCounter+=$Accesses; $DoWTotalVisitsCounter+=$Sessions; $DoWTotalPViewsCounter+=$PViews; next; } elsif ($logsegment eq "hourly") { next if ($TotalReset); ($Accesses,$Bytes,$Sessions,$PViews,$Hour) = m#^\s*([\d,]+)\s+([\d,]+)\s+([\d,]+|-)\s+([\d,]+)\s+(\d\d)#o; $Accesses =~ s/,//g; $Bytes =~ s/,//g; $Sessions =~ s/,//g; $PViews =~ s/,//g; $HourFilesCounter{$Hour}=$Accesses; $BytesHour{$Hour}=$Bytes; $VisitsHour{$Hour}=$Sessions; $PViewsHour{$Hour}=$PViews; $HourlyTotalBytesCounter+=$Bytes; $HourlyTotalHitsCounter+=$Accesses; $HourlyTotalVisitsCounter+=$Sessions; $HourlyTotalPViewsCounter+=$PViews; next; } elsif ($logsegment eq "records") { $line=$_; if ($line =~ m#Most Hits#o) { ($RecordHits,$RecordHitsDate) = $line=~m#([\d,]*)\s\(([^)]*)\)#o; $RecordHits =~ s/,//g; } elsif ($line =~ m#Most Bytes#o) { ($RecordBytes,$RecordBytesDate) = $line=~m#([\d,]*)\s\(([^)]*)\)#o; $RecordBytes =~ s/,//g; } elsif ($line =~ m#Most Visits#o) { ($RecordVisits,$RecordVisitsDate) = $line=~m#([\d,]*)\s\(([^)]*)\)#o; $RecordVisits =~ s/,//g; } elsif ($line =~ m#Most PViews#o) { ($RecordPViews,$RecordPViewsDate) = $line=~m#([\d,]*)\s\(([^)]*)\)#o; $RecordPViews =~ s/,//g; } next; } elsif ($logsegment eq "files") { next if ($TotalReset); ($Hour,$Minute,$Second,$Day,$Month,$Year, $Accesses,$Bytes,$FileName)= m#^\s*(\d\d):(\d\d):(\d\d) (\d\d) (\w\w\w)\w* (\d\d\d\d) \s*([\d,]+)\s+([\d,]+)\s+(\S.*)#o; next unless ($Month =~ /$CurrMonth/); next if ($FileName =~ /\/[^\/]*\/\.\.\//); $Accesses =~ s/,//g; $Bytes =~ s/,//g; $HitsFileCounter{$FileName}=$Accesses; $BytesFileCounter{$FileName}=$Bytes; $LastAccessFile{$FileName}= "$Year $MonthToNumber{$Month} $Day $Hour $Minute $Second"; next; } elsif ($logsegment eq "404") { next if ($TotalReset); ($Hour,$Minute,$Second,$Day,$Month,$Year, $Accesses,$Bytes,$FileName)= m#^\s*(\d\d):(\d\d):(\d\d) (\d\d) (\w\w\w)\w* (\d\d\d\d) \s*([\d,]+)\s+([\d,]+)\s+(\S.*)#o; next unless ($Month =~ /$CurrMonth/); next if ($FileName =~ /\/[^\/]*\/\.\.\//); $Accesses =~ s/,//g; $Bytes =~ s/,//g; $fnfHitsFileCounter{$FileName}=$Accesses; $fnfBytesFileCounter{$FileName}=$Bytes; $fnfLastAccessFile{$FileName}="$Year $MonthToNumber{$Month} $Day $Hour $Minute $Second"; next; } elsif ($logsegment eq "userids") { next if ($TotalReset); ($Hour,$Minute,$Second,$Day,$Month,$Year, $Accesses,$Bytes,$UserID)= m#^\s*(\d\d):(\d\d):(\d\d) (\d\d) (\w\w\w)\w* (\d\d\d\d) \s*([\d,]+)\s+([\d,]+)\s+(\S.*)#o; next unless ($Month =~ /$CurrMonth/); $Accesses =~ s/,//g; $Bytes =~ s/,//g; $HitsUserIDCounter{$UserID}=$Accesses; $BytesUserIDCounter{$UserID}=$Bytes; $LastAccessUserID{$UserID}= "$Year $MonthToNumber{$Month} $Day $Hour $Minute $Second"; next; } elsif ($logsegment eq "topleveldomain") { next if ($TotalReset); ($Hour,$Minute,$Second,$Day,$Month,$Year, $Accesses,$Bytes,$TopDomain)= m#^\s*(\d\d):(\d\d):(\d\d) (\d\d) (\w\w\w)\w* (\d\d\d\d) \s*([\d,]+)\s+([\d,]+)\s+(\w+)\s+=\s+#o; next unless ($Month =~ /$CurrMonth/); $Accesses =~ s/,//g; $Bytes =~ s/,//g; $TopDomainFilesCounter{$TopDomain}=$Accesses; $TopDomainBytesCounter{$TopDomain}=$Bytes; $TopDomainAccess{$TopDomain}="$Year $MonthToNumber{$Month} $Day $Hour $Minute $Second"; next; } elsif ($logsegment eq "domains") { next if ($TotalReset); ($Hour,$Minute,$Second,$Day,$Month,$Year, $Accesses,$Bytes,$Domain)= m#^\s*(\d\d):(\d\d):(\d\d) (\d\d) (\w\w\w)\w* (\d\d\d\d) \s*([\d,]+)\s+([\d,]+)\s+(\S.*)#o; next unless ($Month =~ /$CurrMonth/); $Accesses =~ s/,//g; $Bytes =~ s/,//g; $DomainsFilesCounter{$Domain}=$Accesses; $DomainsBytesCounter{$Domain}=$Bytes; $LastAccessDomain{$Domain}="$Year $MonthToNumber{$Month} $Day $Hour $Minute $Second"; next; } close (OLD); } } sub ReadLog { if ($Verbose) { print " Reading Log File: $LogFile\n"; } if ($IPLog) { dbmopen(%Resolved,$IPLog,0644); } open (LOGFILE,"$LogFile") || die " Error Opening File: $LogFile\n"; $LogLineTotal = 0; $LogLineCount = 0; $ResolutionCount = 0; $logtype = ""; while() { $LogLineTotal++; chop; s/\s+/ /go; next if m/format\=\%/i; if (m/^#fields:/i) { $logtype = "microsoft"; @logtype = split(/\s/,$_); foreach $field (1..(@logtype-1)) { if ($logtype[$field] eq "date") { $MSdate = $field-1; } if ($logtype[$field] eq "time") { $MStime = $field-1; } if ($logtype[$field] eq "c-ip") { $MSdomain = $field-1; } if ($logtype[$field] eq "cs-method") { $MSrequest1 = $field-1; } if ($logtype[$field] eq "cs-uri-stem") { $MSrequest2 = $field-1; } if ($logtype[$field] eq "cs-uri-query") { $MSrequest3 = $field-1; } if ($logtype[$field] eq "sc-status") { $MSstatus = $field-1; } if ($logtype[$field] eq "sc-bytes") { $MSbytes = $field-1; } if ($logtype[$field] eq "cs(User-Agent)") { $MSagent = $field-1; } if ($logtype[$field] eq "cs(Referer)") { $MSreferer = $field-1; } } unless ($MSagent) { $AgentsFile = ""; } unless ($MSreferer) { $RefsFile = ""; } } next if m/^#/; $referer = ""; $Agent = ""; if ($logtype eq "microsoft") { @elements = split(/\s/,$_); ($Year,$Month,$Day) = $elements[$MSdate] =~ m/^(\d+)-(\d+)-(\d+)/o; $Time = $elements[$MStime]; $TimeDate = "$Day/$NumberToMonth{$Month}/$Year:$Time "; $Domain = $elements[$MSdomain]; if ($MSrequest1) { $Request = $elements[$MSrequest1]; } else { $Request = "GET"; } $Request .= " $elements[$MSrequest2]"; if ($MSrequest3) { if (length($elements[$MSrequest3]) > 1) { $Request .= "?$elements[$MSrequest3]"; } } $Request .= " HTTP/1.0"; $Status = $elements[$MSstatus]; $Bytes = $elements[$MSbytes]; if ($MSagent) { $Agent = $elements[$MSagent]; $Agent =~ s/\+/ /g; if (length($Agent) < 5) { $Agent = ""; } } if ($MSreferer) { $referer = $elements[$MSreferer]; if (length($referer) < 5) { $referer = ""; } } $rfc931 = "-"; $authuser = "-"; } elsif (($Domain,$rfc931,$authuser,$TimeDate,$Request,$Status,$Bytes,$referer,$Agent) = /^(\S+) (\S+) (.+) \[(.+)\] \"(.+)\" (\S+) (\S+) \"(.*)\" \"(.*)\"/o) { $logtype = "combined"; if (length($referer) < 5) { $referer = ""; } if (length($Agent) < 5) { $Agent = ""; } } elsif (($Domain,$rfc931,$authuser,$TimeDate,$Request,$Status,$Bytes) = /^(\S+) (\S+) (.+) \[(.+)\] \"(.+)\" (\S+) (\S+)/o) { unless ($logtype) { $logtype = "standard"; } } else { next; } unless ($Agent =~ /^\w/) { $Agent = ""; } next if (!($Domain && $rfc931 && $authuser && $TimeDate && $Request && $Status)); ($Day,$Month,$Year,$Hour,$Minute,$Second)=$TimeDate=~ m#^(\d+)/(\w\w\w)/(\d\d\d\d):(\d\d):(\d\d):(\d\d) #o; if ($HourOffset) { &date_to_count(int($MonthToNumber{$Month}),int($Day),($Year-1900)); $Hour += $HourOffset; if ($Hour < 0) { $Hour += 24; $perp_days -= 1; &count_to_date($perp_days); $Year = $perp_year+1900; $Month = $perp_mon; if (length($Month) == 1) { $Month = "0".$Month; } $Month = $NumberToMonth{$Month}; $Day = $perp_day; } if ($Hour > 23) { $Hour -= 24; $perp_days += 1; &count_to_date($perp_days); $Year = $perp_year+1900; $Month = $perp_mon; if (length($Month) == 1) { $Month = "0".$Month; } $Month = $NumberToMonth{$Month}; $Day = $perp_day; } } if (length($Hour) == 1) { $Hour = "0".$Hour; } if (length($Day) == 1) { $Day = "0".$Day; } $Today="$Year $MonthToNumber{$Month} $Day $Hour $Minute $Second"; next if ($EOMDate && ($Today gt $EOMDate)); $EndDate=$Today if ($Today gt $EndDate); $FileEndDate=$Today if ($Today gt $FileEndDate); $FileStartDate=$Today if ($Today lt $FileStartDate); next unless (!$LogOnlyNew || ($Today gt $InitialEndDate)); ($Method,$FileName,$Protocol)=split(/\s/,$Request,3); $FileName=~ s/\%7[eE]/~/o; $FileName=~ s#//#/#go; $FileName=~ s#\.((s|p)*html*)/.*$#\.$1#o; $FileName=~ s#\.(cgi)/.*$#\.$1#o; $FileName=~ s#\.(pl)/.*$#\.$1#o; $FileName=~ s#^ *$#/#o; $FileName=~ s/#.+$//o; while ($FileName =~ /\/[^\/]*\/\.\.\//) { $FileName =~ s/\/[^\/]*\/\.\.\//\//; } $target = $FileName; unless ($IncludeQuery) { $FileName=~s#\?.*$##o; } $FileName=~ s/${DefaultPageName}$//oi; next if (($IncludeOnlyRefsTo) && !($FileName=~m#$IncludeOnlyRefsTo#o)); next if (($ExcludeRefsTo) && ($FileName=~m#$ExcludeRefsTo#o)); $FileName=~s#&#\&\;#go; $FileName=~s#<#\<\;#go; $FileName=~s#>#\>\;#go; $FileName=~s#"#\"#go; $TrimmedDomain = $Domain; if (($Domain =~ /\d+\.\d+\.\d+\.\d+/) && !($NoResolve)) { $TrimmedDomain =~ s/(\d+\.\d+\.\d+)\.\d+/$1\.XXX/; if ($Resolved{$TrimmedDomain}) { $Domain = $Resolved{$TrimmedDomain}; } else { $ResolutionCount++; $Resolved{$TrimmedDomain} = &addr_to_host($Domain); $Resolved{$TrimmedDomain} = &TrimDomain($Resolved{$TrimmedDomain}); unless ($Resolved{$TrimmedDomain}) { $Resolved{$TrimmedDomain} = "unresolved"; } $Domain = $Resolved{$TrimmedDomain}; } $TrimmedDomain = "$Domain ($TrimmedDomain)"; } else { if ($Domain =~ /\d$/) { $TrimmedDomain =~ s/(\d+\.\d+\.\d+)\.\d+/$1\.XXX/; $Domain = "unresolved"; $TrimmedDomain = "$Domain ($TrimmedDomain)"; } else { $TrimmedDomain = $Domain; if ($PrintDomains eq "2") { $Domain = &TrimDomain($Domain); $TrimmedDomain = "$Domain ($TrimmedDomain)"; } } } next if (($IncludeOnlyDomain) && !($Domain=~m#$IncludeOnlyDomain#o)); next if (($ExcludeDomain) && ($Domain=~m#$ExcludeDomain#o)); $PViewFilter = 0; if ((($Status>199)&&($Status<300))||($Status==304)) { unless ($DetailsFilter && ($FileName=~m#$DetailsFilter#oi)) { $PViewFilter = 1; } } $Domain=~tr/A-Z/a-z/; $TrimmedDomain=~tr/A-Z/a-z/; if ($PrintUserIDs && (length($authuser) > 1)) { $UserID = $authuser." (".$Domain.")"; } if (!($EOMDate) && $referer && $RefsFile) { $target=~ s/\?.*$//o; $target=~ s/${DefaultPageName}$//oi; $target = &Simplify($target); $referer=~ s/\%7[eE]/~/o; $referer=~ s#^(http://[^/]+):80/#$1/#o; $refdomain = ""; $refquery = ""; if ($referer =~ m#\?(.+)#o) { $refquery = $1; } $referer=~ s/#.+$//o; $referer=~ s/\?.*$//o; $referer=~ s/${DefaultPageName}$//oi; $referer = &Simplify($referer); unless (($refsexcludefrom && ($referer =~ m#$refsexcludefrom#oi)) || ($refsexcludeto && ($target =~ m#$refsexcludeto#oi))) { $keyform=$target.' '.$referer; $keyform=~ s#\&#\&\;#og; $keyform=~ s#<#\<\;#og; $keyform=~ s#>#\>\;#og; $keyform=~ s#"#\"\;#og; $referer=~ s#\&#\&\;#og; $referer=~ s#<#\<\;#og; $referer=~ s#>#\>\;#og; $referer=~ s#"#\"\;#og; $TargetCounter{$keyform}++; } else { $referer = ""; } if ($referer =~ m#^.+//([\w|\.|-]+)#o) { $refdomain = $1; } unless ($refdomain =~ /\d$/) { if ($refdomain =~ /([^.]*\.[^.]{3,})$/) { $refdomain{$1}++; } elsif ($refdomain =~ /([^.]*\.[^.]{1,3}\.[^.]*)$/) { $refdomain{$1}++; } elsif ($refdomain =~ /([^.]*\.[^.]*)$/) { $refdomain{$1}++; } } if ($referer && $refquery) { $refquery =~ tr/A-Z/a-z/; $refquery =~ s/\%.{2}/ /gi; $refquery =~ s/\+/ /g; $refquery =~ s/\-/ /g; $refquery =~ s/\s+/ /g; $refquery =~ s/\s$//g; $refquery =~ s/\s&/&/g; $refquery =~ s/\s;/;/g; $refquery =~ s/xspc/ /g; if (($refdomain =~ /altavista/i) && ($refquery =~ /\&q=\s*([^&;]+)/i)) { $altavista{$1}++; } elsif ($refdomain =~ /\.aol\./i) { if (($refquery =~ /search=\s*([^&;]+)/i) || ($refquery =~ /\&s=\s*([^&;]+)/i) || ($refquery =~ /\query=\s*([^&;]+)/i)) { $netfind{$1}++; } } elsif (($refdomain =~ /aj\.com/i) || ($refdomain =~ /ask\.com/i) || ($refdomain =~ /askjeeves/i)) { if ($refquery =~ /ask=\s*([^&;]+)/i) { $askjeeves{$1}++; } } elsif (($refdomain =~ /cnet/i) && ($refquery =~ /query=\s*([^&;]+)/i)) { $cnet{$1}++; } elsif (($refdomain =~ /dogpile/i) && ($refquery =~ /q=\s*([^&;]+)/i)) { $dogpile{$1}++; } elsif (($refdomain =~ /euroferret/i) && ($refquery =~ /p=\s*([^&;]+)/i)) { $euroferret{$1}++; } elsif (($refdomain =~ /euroseek/i) && ($refquery =~ /query=\s*([^&;]+)/i)) { $euroseek{$1}++; } elsif (($refdomain =~ /magellan/i) || ($refdomain =~ /mckinley/i)) { if (($refquery =~ /search=\s*([^&;]+)/i) || ($refquery =~ /\&s=\s*([^&;]+)/i)) { $magellan{$1}++; } } elsif ($refdomain =~ /excite/i) { if (($refquery =~ /search=\s*([^&;]+)/i) || ($refquery =~ /\&s=\s*([^&;]+)/i)) { $excite{$1}++; } } elsif ($refdomain =~ /google/i) { if (($refquery =~ /q=\s*([^&;]+)/i) || ($refquery =~ /query=\s*([^&;]+)/i)) { $google{$1}++; } } elsif (($refdomain =~ /goto/i) && ($refquery =~ /keywords=\s*([^&;]+)/i)) { $goto{$1}++; } elsif (($refdomain =~ /hotbot/i) && ($refquery =~ /mt=\s*([^&;]+)/i)) { $hotbot{$1}++; } elsif (($refdomain =~ /infoseek/i) || ($refdomain =~ /go\.com/i)) { $iseek = ""; if ($refquery =~ /qt=\s*([^&;]+)/i) { $iseek .= $1; } if ($refquery =~ /oq=\s*([^&;]+)/i) { $iseek .= " ".$1; } $iseek =~ s/^\s+//g; $iseek =~ s/\s+/ /g; if ($iseek) { $infoseek{$iseek}++; } } elsif (($refdomain =~ /looksmart/i) && ($refquery =~ /key=\s*([^&;]+)/i)) { $looksmart{$1}++; } elsif (($refdomain =~ /lycos/i) && ($refquery =~ /query=\s*([^&;]+)/i)) { $lycos{$1}++; } elsif (($refdomain =~ /mamma/i) && ($refquery =~ /query=\s*([^&;]+)/i)) { $mamma{$1}++; } elsif (($refdomain =~ /metacrawler/i) || ($refdomain =~ /go2net/i)) { if ($refquery =~ /general=\s*([^&;]+)/i) { $metacrawler{$1}++; } } elsif (($refdomain =~ /msn/i) && ($refquery =~ /mt=\s*([^&;]+)/i)) { $msn{$1}++; } elsif (($refdomain =~ /netscape/i) && ($refquery =~ /search=\s*([^&;]+)/i)) { $netscape{$1}++; } elsif (($refdomain =~ /northernlight/i) && ($refquery =~ /qr=\s*([^&;]+)/i)) { $northernlight{$1}++; } elsif (($refdomain =~ /planetsearch/i) && ($refquery =~ /text=\s*([^&;]+)/i)) { $planetsearch{$1}++; } elsif (($refdomain =~ /savvysearch/i) && ($refquery =~ /q=\s*([^&;]+)/i)) { $savvysearch{$1}++; } elsif (($refdomain =~ /snap/i) && ($refquery =~ /keyword=\s*([^&;]+)/i)) { $snap{$1}++; } elsif ($refdomain =~ /webcrawler/i) { if (($refquery =~ /search=\s*([^&;]+)/i) || ($refquery =~ /searchtext=\s*([^&;]+)/i) || ($refquery =~ /text=\s*([^&;]+)/i)) { $webcrawler{$1}++; } } elsif (($refdomain =~ /yahoo/i) && ($refquery =~ /p=\s*([^&;]+)/i)) { $yahoo{$1}++; } elsif (($refquery =~ /general=\s*([^&;]+)/i) || ($refquery =~ /key=\s*([^&;]+)/i) || ($refquery =~ /keyword=\s*([^&;]+)/i) || ($refquery =~ /keywords=\s*([^&;]+)/i) || ($refquery =~ /query=\s*([^&;]+)/i) || ($refquery =~ /search=\s*([^&;]+)/i) || ($refquery =~ /searchstring=\s*([^&;]+)/i) || ($refquery =~ /searchtext=\s*([^&;]+)/i) || ($refquery =~ /term=\s*([^&;]+)/i) || ($refquery =~ /terms=\s*([^&;]+)/i) || ($refquery =~ /text=\s*([^&;]+)/i) || ($refquery =~ /topic=\s*([^&;]+)/i)) { unless ($1 =~ /^[\d\.]*$/) { $othersearch{$1}++; } } } } if (!($EOMDate) && $Agent && $AgentsFile) { unless ($AgentsIgnore && ($FileName=~m#$AgentsIgnore#oi)) { $refscounter++; $Agent =~ s/\&/\&\;/go; $Agent =~ s/"/\"\;/go; $Agent =~ s//\>\;/go; ($longagent,$other) = split(/ [ \n]/,$Agent,2); $agentcounter{$longagent}++; } } if ($Bytes eq "-") { $Bytes = 0; } if ($Status<400) { $Bytes+=250; } else { $Bytes+=500; } $HourFilesCounter{$Hour}++; if ($BytesHour{$Hour}) { $BytesHour{$Hour}+=$Bytes; } else { $BytesHour{$Hour} = $Bytes; } &date_to_count(int($MonthToNumber{$Month}),$Day,($Year-1900)); $DoWFilesCounter{$perp_date}++; if ($DoWBytesCounter{$perp_date}) { $DoWBytesCounter{$perp_date}+=$Bytes; } else { $DoWBytesCounter{$perp_date} = $Bytes; } $MonthlyFilesCounter{"$Month $Year"}++; if ($MonthlyBytesCounter{"$Month $Year"}) { $MonthlyBytesCounter{"$Month $Year"}+=$Bytes; } else { $MonthlyBytesCounter{"$Month $Year"} = $Bytes; } $Today="$Year $MonthToNumber{$Month} $Day"; $DayFilesCounter{$Today}++; if ($BytesDay{$Today}) { $BytesDay{$Today}+=$Bytes; } else { $BytesDay{$Today} = $Bytes; } if ($PViewFilter) { $MonthlyPViewsCounter{"$Month $Year"}++; $MonthlyTotalPViewsCounter++; $PViewsDay{$Today}++; $DoWPViewsCounter{$perp_date}++; $DoWTotalPViewsCounter++; $PViewsHour{$Hour}++; $HourlyTotalPViewsCounter++; } $Today="$Today $Hour $Minute $Second"; $TopDomain=&GetTopDomain($Domain); if ($Month =~ /$CurrMonth/) { if ($PrintDomains) { $DomainsFilesCounter{$Domain}++; if ($DomainsBytesCounter{$Domain}) { $DomainsBytesCounter{$Domain}+=$Bytes; } else { $DomainsBytesCounter{$Domain} = $Bytes; } unless ($LastAccessDomain{$Domain}) { $LastAccessDomain{$Domain} = 0; } if ($LastAccessDomain{$Domain} lt $Today) { $LastAccessDomain{$Domain}=$Today; } } $TopDomainFilesCounter{$TopDomain}++; if ($TopDomainBytesCounter{$TopDomain}) { $TopDomainBytesCounter{$TopDomain}+=$Bytes; } else { $TopDomainBytesCounter{$TopDomain} = $Bytes; } if ($TopDomainAccess{$TopDomain} lt $Today) { $TopDomainAccess{$TopDomain} = $Today; } } $MonthlyTotalBytesCounter+=$Bytes; $MonthlyTotalHitsCounter++; if ($DayBreak-$perp_days < 35) { $DailyTotalBytesCounter+=$Bytes; $DailyTotalHitsCounter++; if ($PViewFilter) { $DailyTotalPViewsCounter++; } } $DoWTotalBytesCounter+=$Bytes; $DoWTotalHitsCounter++; $HourlyTotalBytesCounter+=$Bytes; $HourlyTotalHitsCounter++; $TodayBytes+=$Bytes; if ($PrintUserIDs && (length($authuser) > 1)) { if ($Month =~ /$CurrMonth/) { if ($PrintUserIDs) { $HitsUserIDCounter{$UserID}++; if ($BytesUserIDCounter{$UserID}) { $BytesUserIDCounter{$UserID}+=$Bytes; } else { $BytesUserIDCounter{$UserID} = $Bytes; } if ($LastAccessUserID{$UserID} lt $Today) { $LastAccessUserID{$UserID}=$Today; } } } } if ($Status==404) { if ($Month =~ /$CurrMonth/) { if ($Print404) { $fnfHitsFileCounter{$FileName}++; if ($fnfBytesFileCounter{$FileName}) { $fnfBytesFileCounter{$FileName}+=$Bytes; } else { $fnfBytesFileCounter{$FileName} = $Bytes; } if ($fnfLastAccessFile{$FileName} lt $Today) { $fnfLastAccessFile{$FileName}=$Today; } } } } unless ((($Status>199)&&($Status<300))||($Status==304)) { $FileName = $RespCodes{$Status}; } if ($Month =~ /$CurrMonth/) { if ($PrintFiles) { $HitsFileCounter{$FileName}++; if ($BytesFileCounter{$FileName}) { $BytesFileCounter{$FileName}+=$Bytes; } else { $BytesFileCounter{$FileName} = $Bytes; } if ($LastAccessFile{$FileName} lt $Today) { $LastAccessFile{$FileName}=$Today; } } } unless ($RefsFile) { $referer = "-"; } unless ($referer) { $referer = "-"; } unless ($AgentsFile) { $Agent = "-"; } unless ($Agent) { $Agent = "-"; } unless ((($Status>199)&&($Status<300))||($Status==304)) { if (($Status>299)&&($Status<400)) { $TodayHits++; } else { $TodayErrors++; } unless ($DetailsFilter && ($FileName=~m#$DetailsFilter#oi)) { unless ($NoSessions) { if (!($EOMDate) && $DetailsFile) { push (@Accesses, "$Day`$Month`$perp_days`$Hour`$Minute`$Second`$TrimmedDomain`$FileName`$referer`$Agent"); } else { push (@Accesses, "$perp_days`$Hour`$Minute`$Second`$TrimmedDomain"); } } } } else { $TodayHits++; if ($PViewFilter) { if ($OrgDomain && ($Domain !~ /nresolve/)) { if (($Domain !~ /\./) || ($Domain =~ /.*$OrgDomain/)) { $TodayLocal++; } } $TodayPages++; $TodayDomains{$TopDomain}++; $TodayHosts{$TrimmedDomain}++; $TodayPages{$FileName}++; unless ($NoSessions) { if (!($EOMDate) && $DetailsFile) { push (@Accesses, "$Day`$Month`$perp_days`$Hour`$Minute`$Second`$TrimmedDomain`$FileName`$referer`$Agent"); } else { push (@Accesses, "$perp_days`$Hour`$Minute`$Second`$TrimmedDomain"); } } } } $LogLineCount++; } close (LOGFILE); if ($IPLog) { dbmclose(%Resolved); } unless (($logtype eq "combined") || ($logtype eq "microsoft")) { unless ($logtype eq "standard") { if ($Verbose) { print " Error: Unrecognized access log format\n"; } return; } $RefsFile = ""; $AgentsFile = ""; } if ($Verbose) { print " ",&commas($LogLineCount)," of ",&commas($LogLineTotal)," log entries processed\n"; } if ($Verbose) { print " ",&commas($ResolutionCount)," IP addresses resolved\n"; } } sub GetTopDomain { local($domname) = $_[0]; ($TopDomain)=($domname=~m#\.(\w+)$#o); $TopDomain=~tr/A-Z/a-z/; $TopDomain=~s/\d//g; $TopDomain="xxx" if ($TopDomain eq ""); unless ($CountryCodes{$TopDomain}) { $TopDomain = "ooo"; } return $TopDomain; } sub addr_to_host { local($ip_address) = $_[0]; local(@bytes) = split(/\./, $ip_address); local($packaddr) = pack("C4",@bytes); local($host_name) = (gethostbyaddr($packaddr, 2))[0]; return($host_name); } sub TrimDomain { local($trimresult) = $_[0]; $trimresult =~ s/[^\.]*\.(.*)/$1/; if ($PrintDomains eq "2") { if ($trimresult =~ /([^.]*\.[^.]{3,})$/) { $trimresult = $1; } elsif ($trimresult =~ /([^.]*\.[^.]{1,3}\.[^.]*)$/) { $trimresult = $1; } elsif ($trimresult =~ /([^.]*\.[^.]*)$/) { $trimresult = $1; } } return $trimresult; } sub PrintReport { open (REPORT,">$FileDir/$ReportFile") || die " Error Opening File: $ReportFile\n"; print REPORT "Access Log: $SystemName"; print REPORT "\n"; if ($headerfile) { open (HEADER,"$headerfile"); @header =
; close (HEADER); foreach $line (@header) { print REPORT "$line"; } } print REPORT "

Access Log:
$SystemName

\n"; unless ($EndDate eq "0000 00 00 00 00 00") { print REPORT "

(Accesses Through "; ($Year,$Month,$Day,$Hour,$Minute,$Second)= split(/ /,$EndDate); print REPORT "$Hour:$Minute:$Second $Day "; print REPORT "$NumberToMonth{$Month} $Year)

\n"; } if (!($EOMDate) && ($DetailsFile || $RefsFile || $KeywordsFile || $AgentsFile)) { print REPORT "

|"; if ($DetailsFile) { print REPORT " Access Details Report |"; } if ($RefsFile) { print REPORT " Referring URLs Report |"; } if ($KeywordsFile) { print REPORT " Keywords Report |"; } if ($AgentsFile) { print REPORT " Agents/Platforms Report |"; } print REPORT "

\n"; } print REPORT "
\n"; print REPORT "

Index

\n"; print REPORT "

\n"; print REPORT "
\n"; print REPORT "

Long-Term Statistics:

     

"; print REPORT "		\n"; print REPORT "

Statistics for the Current Month ($CurrMonth $CurrYear):

\n"; print REPORT "

\n"; print REPORT "\n"; print REPORT "

Key Terms:\n"; print REPORT "

Hits: The total number of files requested from the server.\n"; print REPORT "
Bytes: The amount of information transferred in filling those requests.\n"; print REPORT "
Visits: The (approximate) number of actual individual visitors.\n"; print REPORT "
PViews: The number of Web pages viewed by those visitors.\n"; print REPORT "

The bar graphs below are based upon "; if ($GraphBase eq "hits") { print REPORT "number of hits"; } elsif ($GraphBase eq "visits") { print REPORT "number of visitors"; } elsif ($GraphBase eq "pviews") { print REPORT "page views"; } else { print REPORT "bytes transferred"; } print REPORT ".

\n"; &PrintMonthlyReport; &PrintDailyReport; &PrintDayofWeekReport; &PrintHourlyReport; &PrintRecords; if ($PrintFiles) { if ($PrintTopNFiles) { &PrintTopNFilesByHitsReport; &PrintTopNFilesByVolumeReport; } &PrintFilesReport; } if ($Print404) { if ($PrintTopNFiles) { &PrintTopN404Report; } &Print404Report; } if ($PrintUserIDs) { &PrintUserIDsReport; } &PrintTopLevelDomainsReport; if ($PrintDomains) { if ($PrintTopNDomains) { &PrintTopNDomainsByHitsReport; &PrintTopNDomainsByVolumeReport; } &PrintReversedDomainsReport; } print REPORT "
\n"; if (!($EOMDate) && ($DetailsFile || $RefsFile || $KeywordsFile || $AgentsFile)) { print REPORT "

|"; if ($DetailsFile) { print REPORT " Access Details Report |"; } if ($RefsFile) { print REPORT " Referring URLs Report |"; } if ($KeywordsFile) { print REPORT " Keywords Report |"; } if ($AgentsFile) { print REPORT " Agents/Platforms Report |"; } print REPORT "

\n"; } print REPORT "

"; print REPORT "This report was generated by "; print REPORT ""; print REPORT "WebLog $version

\n"; if ($footerfile) { open (FOOTER,"$footerfile"); @footer =