functions.q

/=====================================================================================
/      Filename:  functions.q
/   Description:  Script contains the main functions used to scan a file by Q
/       Version:  1.0
/       Created:  28/01/12 23:00:55
/        Author:  Oliver Fletcher, ttolf@lboro.ac.uk
/    University:  Loughborough University
/=====================================================================================

load `:database/virus_sigs;              / Load in the database tables
load `:database/md5_sigs;
load `:database/filetype_sigs;
load `:database/filetype_iden;
load `:database/settings;

regexb:`boost 2:(`regexp;2);              / Load the boost regex shared object file

/ ===  FUNCTION  ======================================================================
/        Name:    vregex
/ Description:    Takes an input string, or vector of strings and compares to a regular
/                 expression.
/      Inputs:    str         -> a string to be matched against patterns
/                 expressions -> a single/list of regular expressions
/     Returns:    for a single expression -> 0 or 1
/                 for a vector of expressions -> a vector of 0 and 1's
/ =====================================================================================
vregex:{[str;expression]
  $[0h~type expression;regexb[str;]each expression;regex[str;expression]]}

/ ===  FUNCTION  ======================================================================
/        Name:    load_file
/ Description:    Reads a file into Q as binary data.
/                 Grabs the file extension
/                 Converts the hex output to string and takes md5 value
/                 Queries the filetype database attempting to recognise file type
/      Inputs:    file -> path to a file to be read
/     Outputs:    ext -> file type extension
/                 file_hex -> string of files binary data
/                 filetype -> Table of file type information discovered
/ =====================================================================================
load_file:{[file]
  strip:{[x](neg (count x) - (1 + last where x=".")) sublist x};
  ext::@[strip;file;"Unknown"];                            / Get the files extension

  lfile:":",file;                                          / Read the file in as binary
  hex_dump:read1(`$lfile);

  $[hex_dump~`byte$();hex_dump:4h$"Empty File";];          / Label an empty file

  file_hex::raze string hex_dump;                          / Convert to string

  file_md5::raze string md5 file_hex;                      / Get md5 value

  / Atempt to get the filetype using the filetype signature database
  filetype::filetype_sigs[where vregex[file_hex;filetype_sigs[`Sig]]];
  $[(count filetype)>0;filetype::filetype[0];filetype::-1#filetype_sigs];
  fname::select from filetype_iden where Id=filetype[`fileType];
  }

/ ===  FUNCTION  ======================================================================
/        Name:    qupdate
/ Description:    Runs the update.q script, timing it and counting how many signatures
/                 and md5 hashes were added to the database
/      Inputs:
/     Outputs:    Time taken for update and signatures updated
/ =====================================================================================
qupdate:{
  start:.z.t;                                              / Start timer
  start_v:count virus_sigs;                                / Count database signatures
  start_m:count md5_sigs;

  value"\\l qupdate/qupdate.q";                            / Run qupdate.q script

  finish:.z.t;
  finish_v:count virus_sigs;
  finish_m:count md5_sigs;

  out: "Total time taken: ", (string finish-start);        / Save output data
  out:(out;"MD5 Signatures Updated: ", (string finish_m-start_m));
  out,: "Malware Signatures Updated: ", (string finish_v-start_v);
  out
  }

/ ===  FUNCTION  ======================================================================
/        Name:    qscan
/ Description:    Changes Q into the directory provided
/                 Creates a handle from filename and directory
/                 Gets directory files (not including directory)
/                 Scans the system using the fscan function
/      Inputs:    filename -> file to be scanned
/                 dir      -> path/to/file/
/     Outputs:    output   -> log information on the scan
/ =====================================================================================
qscan:{[filename;dir]
  home:system["pwd"];                                      / Save current directory
  system ["cd ",dir];                                      / Change to scan directory
  path:system["pwd"];                                      / Get current directory path

  /directory::system ["ls -l ",dir," | awk 'NR!=1 && !/(^d|^l)/ {print $NF}'"];

  handle:`$(raze ":",path);                                / Create file handle

  / Get a vector containg just the files in the current directory
  directory: string (key handle)[where -11h~/:type each key each(` sv'handle,/:key handle)];

  / if filename exists then check that the filename exists in directory
  error:$[not ""~filename;not max directory like (filename);0b];
  $[error;'`$"Error Reading File (Possibly does not Exist)";];

  $[""~filename;output:{fscan[x]}each directory;output:fscan[filename]]; / Scan files

  system ["cd ",raze home];                                 / Return to last directory
  output
  }

/ ===  FUNCTION  ======================================================================
/        Name:    fscan
/ Description:    Loads a file into Q using load_file[]
/                 Scans MD5 database
/                 Scans virus sig database
/                 Times scan
/      Inputs:    filename -> file to be scanned
/     Outputs:    out      -> log of scan information
/ =====================================================================================
fscan:{[filename]
  start:.z.t;                                              / Start timer
  load_file[filename];                                     / Load file
  out:("Filename: ",filename);

  $[not 0~count filetype;                                  / Check file exists
    ftype:("Recognised: ", raze fname[`Name]);
    ftype:"Not Recognised"
    ];

  / Log data
  out:(out;("File size: ", (raze string (count file_hex)%2000)," kb"));
  out,:("File extention: .", raze string ext);
  out,:("File MD5: ", raze string file_md5);
  out,:ftype;

  $[settings[`md5];[                                       / If md5 is switched on
      out,:"Scanning md5 database...";
      md5_results:vregex[file_md5;md5_sigs[`MD5]];         / Scan the file

      $[0~max md5_results;                                 / Generate results data
        out,:"MD5 Database: Not Infected";
        [malware:md5_sigs[`MalwareName][where md5_results];out,:raze "Malware Detected: ", string malware;]];
      ];
    [out,:"not scanning";
    out,:"MD5 Database not scanned (turn on in settings)";]];

  $[settings[`virus];[                                      / Virus scanning on
      out,: "Scanning malware signature database (warning may take some time)...";

      / Select the viruses matching the inputted files filetype signature
      $[filetype[`fileType]~10;viruses:virus_sigs;viruses:select from virus_sigs where TargetType=filetype[`fileType]];
      virus_results:vregex[file_hex;viruses[`HexSig]];
      $[0~max virus_results;
        out,: "Malware Database: Not Infected";
        [malware:viruses[where virus_results];out,:raze "Malware Detected: ", string malware[`MalwareName];]];
      ];

    [out,:"not scanning";
    out,:"Virus Signature Database not scanned (turn on in settings)";]
    ];

  / Log information and return the log
  finish:.z.t;
  out,: "Total time taken: ", (string finish-start);
  $[settings[`verbose];out;out[0 6 8]]
  }

/ ===  FUNCTION  ======================================================================
/        Name:    qsettings
/ Description:    Updates the current scan settings
/      Inputs:    updateSettings -> vector of 0's and 1's containing updated settings
/     Outputs:
/ =====================================================================================
qsettings:{[updateSettings]
  settings[`md5`virus`verbose`updates]:1h$(updateSettings);
  save `:database/settings;
  }