Guest User

Untitled

a guest
Dec 7th, 2023
136
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. import { LoggerText, LogLevel } from 'crawlee';
  2. import * as winston from 'winston';
  3. import { fileURLToPath } from 'url';
  4. import * as path from 'path';
  5. import * as fs from 'fs';
  6.  
  7. const __filename = fileURLToPath(import.meta.url);
  8.  
  9. const __dirname = path.dirname(__filename);
  10.  
  11. export const BASE_PATH = path.dirname(path.dirname(__dirname));
  12.  
  13. export type AdditionalData = Record<string, any> | null | unknown;
  14.  
  15. //mapping winston's log levels to crawlee's on index
  16. // DONT CHANGE
  17. const winstonLogLevelMapping = [
  18.     'info',
  19.     'error',
  20.     'warn',
  21.     'warn',
  22.     'info',
  23.     'debug',
  24.     'verbose',
  25. ];
  26.  
  27. export const initWinstonLogger = (label: string) => {
  28.     const { printf } = winston.format;
  29.     const myFormat = printf(({ level, message, timestamp, label }) => {
  30.         return `${timestamp} [${label}] ${level}: ${message}`;
  31.     });
  32.     //creating logs folder if it doesnt exist
  33.     fs.mkdirSync(`${BASE_PATH}/logs`, { recursive: true });
  34.     winston.loggers.add(label,{
  35.         level: "silly",
  36.         format: winston.format.combine(
  37.             winston.format.label({ label: label }),
  38.             winston.format.timestamp(),
  39.             myFormat
  40.         ),
  41.         transports: [
  42.             new winston.transports.Console({
  43.                 format: winston.format.colorize({ all: true }),
  44.             }),
  45.             new winston.transports.File({
  46.                 dirname: `${BASE_PATH}/logs/`,
  47.                 filename: `output.log`,
  48.                 handleExceptions: true,
  49.                 maxsize: 50000000,
  50.                 maxFiles: 10,
  51.                 tailable: true,
  52.                 format: winston.format.uncolorize(),
  53.             }),
  54.         ],
  55.     });
  56.     return winston.loggers.get(label)
  57. };
  58.  
  59. export class CrawlerLogger extends LoggerText {
  60.     winstonLogger: winston.Logger;
  61.     constructor(options: any = {}, label = '') {
  62.         super(options);
  63.         this.winstonLogger = initWinstonLogger(label)
  64.     }
  65.  
  66.     override log(
  67.         level: LogLevel,
  68.         message: string,
  69.         data?: AdditionalData,
  70.         exception?: Error
  71.     ) {
  72.         const errStack = exception ? this._parseException(exception) : '';
  73.         const dataStr = !data ? '' : ` ${JSON.stringify(data, null, 2)}`;
  74.         const logStr = message + errStack + dataStr;
  75.  
  76.         this.winstonLogger.log(winstonLogLevelMapping[level], logStr);
  77.     }
  78. }
  79.  
Advertisement
Add Comment
Please, Sign In to add comment