Untitled

/*=============================================================================
Program Name:         SCANLOG.SAS
Purpose:              To automatically scan specified LOG file for
                      ERROR/WARNING/NOTE information
DataSets Used:        {None}
DataSets Created:     FINDINGS
Macro variables:      INFILE [Default from LOGCHECK macro]:
                             Actual LOG file to scan for ERROR/WARNING/NOTE
Programmer:           Lawrence Heaton-Wright
OS/SAS Vnum:          Windows NT / 8.2
Date:                 29-Aug-2003
Notes:                This macro can be run from interactive or batch SAS.
                      This macro is called from LOGCHECK.
Example:
--------------------------------------------------------------------------------
 Modifications:
 Programmer/Date:     Reason:
 ----------------     -------
 LHW / 29-JAN-2004    Exclude warnings about license expiry.
 LHW / 14-JAN-2005    Added FILEINFO data set step. This will be used to create
                      a summary of files scanned and which ones have issues in
                      them [LOGCHECK macro].
 LHW / 19-MAY-2005    Added "Compression was disabled" to exclude from WARNING
                      messages regarding compression.
 LHW / 24-AUG-2005    Adapt ERESINFO section to accomodate the standardised
                      global directory structure LOG information.
 LHW / 26-SEP-2005    To ensure that SCANLOG can still be used even on non-standard
                      log files ensure that if log stype is non-standard
                      (i.e. not QTHV or globalisation standard) then set this to
                      THV as version 0.2 of SCANLOG worked like this for non-standard files.
 LHW / 09-NOV-2005    Changed length of FILENAME from 132 characters to 200 to
                      accomodate very long file names.
                      Retain TYPE variable for correct scanning of ERES information.
CP   / 07-NOV-2006    keep only notes needed for QSTR
==============================================================================*/

%MACRO ScanLog (InFile=);
    FILENAME LogFile "&InFile";
%*-------------------------------------------------------------------------------*;
%* Read in specified SAS log file - no check in place for confirming existence   *;
%* of file [add later]. Flag lines from LOG which indicate a new page in the SAS *;
%* log and therefore contain a page-throw character. These records to be deleted.*;
%*-------------------------------------------------------------------------------*;
    DATA _Log;
        INFILE LogFile LRECL=200 TRUNCOVER;
        INPUT @1 line $CHAR200.;
        ATTRIB
            ObsNo   LABEL="SAS data set observation number"
        ;
        RETAIN DeleteLine 0;
        ObsNo=_n_;
        IF INDEXW(Line,"The SAS System")>0 AND INDEXW(Line,"NOTE:")=0 THEN DO;
            DeleteLine=ObsNo;
            DelFlag=1;
        END;
        IF ObsNo=DeleteLine+1 THEN DO;
            DelFlag=1;
            DeleteLine=0;
        END;
    RUN;
%*-------------------------------------------------------------------------------*;
%* The format OBSNO is used to convert the number of observations [i.e. number   *;
%* of lines in the log file] to the nearest 100, 1000, etc. This will be used to *;
%* create LINENO for those log file records which are not part of the program    *;
%* code, i.e. NOTEs, WARNINGs, etc.                                              *;
%* Add indicator flags for standard user-defined LOG warning messages            *;
* (i.e. DATA ISSUE:, STL ISSUE: DM ISSUE:).                                      *;
%*-------------------------------------------------------------------------------*;
    PROC FORMAT;
        VALUE obsno
             1-99="100"
          100-999="1000"
        1000-9999="10000"
      10000-99999="100000"
    100000-999999="1000000"
            OTHER=_ERROR_;
    RUN;

    DATA _Log (DROP=WarnTemp ErrorTemp type);
        SET _Log (WHERE=(DelFlag NE 1)) NOBS=nobs;
        ATTRIB
            type        LABEL="Log message type [THV or GBL - Global]"
            LineNo      LABEL="Program line number from LOG file"
            Note        LABEL="LOG line gives NOTE message"
            Warning     LABEL="LOG line gives WARNING message"
            Error       LABEL="LOG line gives ERROR message"
            DataIssue   LABEL="LOG line gives DATA ISSUE message"
            STLIssue    LABEL="LOG line gives STL ISSUE message"
            DMIssue     LABEL="LOG line gives DM ISSUE message"
        ;
        number_obs=nobs;
        RETAIN LineNo ProcDataSets EresInfo Type 0;
%*** Based on the number of observations in the LOGFILE dataset, i.e. number of records in log file ***;
%*** Create an order variable for those log records without a line number ***;
        denom=INPUT(PUT(nobs,obsno.),BEST.);;
        _LineNo=SCAN(line,1," ");
        SecondWord=SCAN(line,2," ");
%*** Standard user-defined LOG messages are always 2 words ending in "ISSUE:" ***;
        IF SecondWord="ISSUE:" THEN
            _LineNo=TRIM(LEFT(_LineNo)) || " " || TRIM(LEFT(SecondWord));

        IF COMPRESS(Line)="-----Directory-----" THEN ProcDataSets=1;
        IF Line="NOTE: PROCEDURE DATASETS used:" THEN ProcDataSets=0;
        IF line IN ("USER INFORMATION:","*  USER & SYSTEM INFORMATION :") THEN EresInfo=1;
        IF line EQ "USER INFORMATION:" THEN type='THV';
        ELSE IF line EQ "*  USER & SYSTEM INFORMATION :" THEN type='GBL';
        IF EresInfo=1 AND SUBSTR(line,1,10) IN ("----------","*---------") THEN EresInfo=0;
        NonNum=0;
        IF INDEXC(_LineNo,'1234567890') THEN DO x=1 TO 255 UNTIL (NonNum=1);
            IF NOT (x=32 OR 48<=x<=57)AND INDEXC(_LineNo,BYTE(x))>0 THEN NonNum=1;
        END;
        IF SecondWord=:"+" THEN NonNum=1;
        IF INDEXC(_LineNo,'1234567890') AND NonNum=0 AND ProcDataSets=0 AND SecondWord NE "at" THEN
            LineNo=INPUT(_LineNo,BEST.);
        ELSE LineNo=LineNo+(1/denom);
%*** Create indicator flags for NOTEs, WARNINGs & ERRORs + standard user-defined messages ***;
        ARRAY SearchVal[8] $11. _TEMPORARY_
            ("NOTE:" "WARNING:" "ERROR:" "WARNING" "ERROR" "DATA ISSUE:" "STL ISSUE:" "DM ISSUE:");
        ARRAY LogVar[*] Note Warning Error WarnTemp ErrorTemp DataIssue STLIssue DMIssue;
        DO i=1 TO DIM(LogVar);
            IF _LineNo=SearchVal[i] THEN LogVar[i]=1;
        END;
        IF WarnTemp=1 THEN Warning=1;
        IF ErrorTemp=1 THEN Error=1;

%*** Checking for non-standard user-defined messages ***;
        %IF &UMessage NE %THEN %DO;
            ARRAY SearchUser[&UserM.] $%EVAL(&MaxUMLen.) _TEMPORARY_ (
                %DO um_i=1 %TO &UserM.;
                    "&&Message&um_i."
                %END;
                );
            ARRAY LogUser[&UserM.];
%*** Check each of the non-standard user-defined messages against the start of LINE ***;
            DO i=1 TO DIM(SearchUser);
                IF Line=:SearchUser[i] THEN LogUser[i]=1;
            END;
        %END;

%*** Translate CR [Carriage return]/FF [Form Feed] characters to spaces ***;
        line=TRANWRD(line,BYTE(13)," ");
        line=TRANWRD(line,BYTE(10)," ");

%*** Non-standard log files use THV standard ERES info scanning ***;
        IF type EQ ' ' THEN type='THV';
        IF type NE ' ' THEN CALL SYMPUT ('LogType',type);
    RUN;
    %PUT Log file type=&LogType.;
%*-------------------------------------------------------------------------------*;
%* Create STEP - an indicator variable with an new integer for each "step" of    *;
%* the log file. Ensure that comment blocks (/* -> */) are included all as one   *;
%* step.                                                                         *;
%*-------------------------------------------------------------------------------*;
    DATA _Step_Log (KEEP=line ObsNo LineNo Step Comment Note Warning Error
                        DataIssue STLIssue DMIssue
                        %IF &UMessage NE %THEN %DO um_i=1 %TO &UserM.;
                            LogUser&um_i.
                        %END;
                        FileName)
         _EresInfo (KEEP=line FileName);
        SET _Log;
        LENGTH Filename $200.;
        RETAIN comment step 0;
        FileName="&InFile";
        IF SecondWord=:"/*" THEN comment=1;
        IF INDEX(line,"*/")>0 THEN comment=2;
        IF comment=2 AND INDEX(line,"*/")=0 THEN comment=0;
        IF (UPCASE(SecondWord) IN ("PROC" "DATA" "OPTIONS") AND note NE 1 AND comment EQ 0) OR
            UPCASE(SecondWord)=:"TITLE" THEN step=step+1;
        OUTPUT _Step_Log;
        IF EresInfo=1 THEN OUTPUT _EresInfo;
    RUN;
    PROC SORT DATA=_Step_Log;
        BY Step LineNo;
    RUN;
%*-------------------------------------------------------------------------------*;
%* Scan through ERES information to find who ran the file and when. Merge this   *;
%* information onto _REPORT prior to checking for LOG problems.                  *;
%* Dependent on LOG file information type [LOGTYPE] create ERESDATA data set     *;
%* based on this type of information. LOGTYPE=THV indicates the QTHV LOG         *;
%* information standard, LOGTYPE=GBL indicates the Global Harmonised Structure   *;
%* Log information standard.                                                     *;
%*-------------------------------------------------------------------------------*;
    %IF &LogType EQ THV %THEN %DO;
        DATA _EresData (KEEP=FileName UserID SubmitDT);
            SET _EresInfo END=eof;
            ATTRIB
                UserID  LABEL="User ID who submitted file" LENGTH=$50.
                _Date   LABEL="Date file submitted [character]" LENGTH=$50.
                _Hour   LABEL="Hour file submitted [character]" LENGTH=$2.
                _Minute LABEL="Minute file submitted [character]" LENGTH=$2. LENGTH=$2.
                SubmitDT    LABEL="Date/Time file submitted" FORMAT=DATETIME20.
            ;
            RETAIN UserID ' ' SubmitDT 0;
            Line=COMPBL(Line);
            IF Line=:"User ID" THEN UserID=SCAN(Line,2,":");
            IF Line=:"Date" THEN DO;
                _Date=SCAN(Line,2,":");
                _Hour=SCAN(COMPRESS(Line),3,":");
                _Minute=SCAN(COMPRESS(Line),4,":");
                SubmitDT=DHMS(INPUT(SCAN(COMPRESS(_Date),2,','),DATE9.),INPUT(_Hour,BEST.),INPUT(_Minute,BEST.),0);
            END;
            IF eof;
        RUN;
    %END;
    %ELSE %DO;
        DATA _EresData (KEEP=FileName UserID SubmitDT);
            SET _EresInfo END=eof;
            ATTRIB
                _QID        LABEL="Quintiles User ID" LENGTH=$7.
                _UserName   LABEL="User Name" LENGTH=$40.
                UserID      LABEL="User ID who submitted file" LENGTH=$50.
                _FullDate   LABEL="Full Date file submitted [character]" LENGTH=$50.
                _Day        LABEL="Day file submitted [character]" LENGTH=$2.
                _Month      LABEL="Month file submitted [character]" LENGTH=$10.
                _Year       LABEL="Year file submitted [character]" LENGTH=$4.
                _Time       LABEL="Time file submitted" FORMAT=TIME8.
                _Date       LABEL="Date file submitted" FORMAT=DATE9.
                SubmitDT    LABEL="Date/Time file submitted" FORMAT=DATETIME20.
            ;
            RETAIN _QID _UserName ' ' _Time _Date 0;
            Line=COMPBL(Line); Line=TRANWRD(Line,': ',':');
            IF Line=:"* User ID" THEN _QID=SCAN(Line,2,":");
            IF Line=:"* User Name" THEN _UserName=SCAN(Line,2,":");
            IF Line=:"* Date" THEN DO;
                _FullDate=SCAN(Line,2,":");
                _Day=COMPRESS(TRANWRD(SCAN(_FullDate,3,' '),',',' '));
                _Month=SUBSTR(SCAN(_FullDate,2,' '),1,3);
                _Year=SCAN(_FullDate,4,' ');
                _Date=INPUT(COMPRESS(_Day || _Month || _Year),DATE9.);
            END;
            IF Line=:"* Time :" THEN _Time=INPUT(SUBSTR(SCAN(Line,3," "),2),TIME8.);
            IF eof THEN DO;
                SubmitDT=DHMS(_Date,HOUR(_Time),MINUTE(_Time),SECOND(_Time));
                UserID=_QID || " (" || TRIM(LEFT(_UserName)) || ")";
                OUTPUT;
            END;
        RUN;
    %END;
%*-------------------------------------------------------------------------------*;
%* Check the NOTE contents for "errors" - NOTE information that could be         *;
%* potential issues, i.e. Unitialized variables, etc.                            *;
%*-------------------------------------------------------------------------------*;
    DATA _Step_Log;
        SET _Step_Log;
        RETAIN Note_N Warning_N Error_N DataIssue_N STLIssue_N DMIssue_N 0;
        ATTRIB
            Note_N      LABEL="NOTE Identifier within STEP"
            Warning_N   LABEL="WARNING Identifier within STEP"
            Error_N     LABEL="ERROR Identifier within STEP"
            DataIssue_N LABEL="DATA ISSUE Identifier within STEP"
            STLIssue_N  LABEL="STL ISSUE Identifier within STEP"
            DMIssue_N   LABEL="DM ISSUE Identifier within STEP"
            Problem     LABEL="Flag to highlight what message to print in LOG report"
        ;
        BY Step LineNo;
        ARRAY LogVar[*] Note Warning Error DataIssue STLIssue DMIssue;
        ARRAY LogVarNum[*] Note_N Warning_N Error_N DataIssue_N STLIssue_N DMIssue_N;
        DO i=1 TO DIM(LogVar);
            IF FIRST.Step THEN LogVarNum[i]=0;
            IF LogVar[i]=1 THEN LogVarNum[i]=LogVarNum[i]+1;
        END;
%*** Checking NOTES: ***;
        IF Note THEN DO;
            IF /*INDEX(Line,"created, with 0 rows")>0 OR INDEX(Line,"has 0 observations")>0 OR*/
                INDEX(Line,"uninitialized")>0 OR INDEX(Line,"Invalid")>0 OR INDEX(Line,"invalid")>0 OR
                INDEX(Line,"Division by zero")>0 /*OR INDEX(Line,"Missing values were generated")>0*/ OR
             /*   INDEX(Line,"Numeric values have been converted to character values")>0 OR
                INDEX(Line,"Character values have been converted to numeric values")>0 OR */
                INDEX(Line,"MERGE statement has more than one data set with repeats of BY values")>0 OR
              /*  INDEX(Line,"At least one W.D format was too small for the number to be printed")>0 OR
                INDEX(Line,"Character format specified for the result of a numeric expression")>0 OR
               */ INDEX(Line,"Mathematical operations could not be performed at the following places")>0 OR
             /*   INDEX(Line,"ERROR DETECTED IN ANNOTATE=")>0 OR INDEX(Line,"PROBLEM IN OBSERVATION")>0 OR
                INDEX(Line,"Reduction in size of titles")>0 OR INDEX(Line,"outside the axis range")>0 OR
               */ INDEX(Line,"The SAS System stopped processing this step because of insufficient disk space")>0 OR
                INDEX(Line,"not found or could not be loaded")>0
                 THEN Problem=1;
        END;
        IF Warning OR Error THEN Problem=1;
        IF DataIssue OR STLIssue OR DMIssue THEN Problem=1;
        IF Warning AND
            (INDEX(Line,"is associated will expire within")>0 OR
             INDEX(Line,"Compression was disabled")>0 OR
             INDEX(Line,"product with which")>0 OR
             INDEX(Line,"Your system is scheduled to expire on")>0) THEN Problem=0;

        %IF &UMessage NE %THEN %DO um_i=1 %TO &UserM.;
            ARRAY LogUser[&UserM.];
            DO i=1 TO DIM(LogUser) UNTIL (Problem=1);
                IF LogUser[i]=1 THEN Problem=1;
            END;
        %END;
    RUN;
%*-------------------------------------------------------------------------------*;
%* Select steps with PROBLEM=1 - need to investigate. Potential problem.         *;
%*-------------------------------------------------------------------------------*;
    PROC SQL NOPRINT;
        CREATE TABLE _Step_Log AS
            SELECT s.*, UserID, SubmitDT
            FROM _Step_Log s LEFT JOIN _EresData e
            ON s.FileName EQ e.FileName;
        CREATE TABLE _Report AS
            SELECT *, INT(LineNo) AS RepLine FROM _Step_Log
            WHERE problem=1 ORDER BY Step, LineNo;
        CREATE TABLE _FileInfo_file AS
            SELECT DISTINCT s.filename, UserID, SubmitDT, Issue
            FROM _Step_Log s LEFT JOIN
                (SELECT DISTINCT filename, 1 AS issue FROM _Step_Log WHERE problem EQ 1) p
            ON s.filename EQ P.filename ORDER BY s.filename;
    QUIT;
%*-------------------------------------------------------------------------------*;
%* Append all _REPORT data sets to FINDINGS: This is the final report.           *;
%*-------------------------------------------------------------------------------*;
    PROC APPEND BASE=_findings FORCE DATA=_Report;
    RUN;
    PROC APPEND BASE=_FileInfo FORCE DATA=_FileInfo_file;
    RUN;
%*-------------------------------------------------------------------------------*;
%* Remove data sets [_LOG/_STEP_LOG/_REPORT] from WORK area at end of SCANLOG.   *;
%*-------------------------------------------------------------------------------*;
    PROC DATASETS LIBRARY=WORK MTYPE=DATA;
        DELETE _log _step_log _report _eres: _FileInfo_file;
    RUN; QUIT;
%MEND ScanLog;