Guest User

example of OOP python main

a guest
Mar 13th, 2022
246
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 3.30 KB | None | 0 0
  1. import logging
  2. import os
  3. import sys
  4. from dataclasses import dataclass
  5. from typing import List
  6. from uuid import uuid4
  7.  
  8. import structlog
  9. from bson.objectid import ObjectId
  10. from dotenv import load_dotenv
  11. from pymongo import uri_parser
  12. from pymongo.collection import Collection
  13. from pymongo.mongo_client import MongoClient
  14.  
  15. from etl import model
  16. from etl.anonymization import AccessorsAnonymizer, Tokenizer
  17. from etl.bigquery import Bigquery, CommandTableFacade
  18. from etl.processing import Processor
  19. from etl.util import fatal
  20.  
  21. FALLBACK_MONGO_URI = "mongodb://localhost:27017/we"
  22. execution_id = str(uuid4())
  23.  
  24.  
  25. def main():
  26.     cfg = get_config()
  27.  
  28.     if cfg.is_deployment:
  29.         configure_logging()
  30.     log = structlog.stdlib.get_logger().bind(execution_id=cfg.execution_id, environment=cfg.environment)
  31.  
  32.     log.info(f"Discovered configuration: {cfg}")
  33.  
  34.     cfg.deployment_config_sanity_check(log)
  35.  
  36.     # MongoCatalog, Tokenizer, AccessorsAnonymizer, Bigquery and CommandTableFacade are "library" classes. Notices that these classes are instantiated only once.
  37.    
  38.     mongo = MongoCatalog(cfg.mongo_rs_core_uri, cfg.mongo_rs_accessors_uri)
  39.     tokenizer = Tokenizer(cfg.tokenizer_url, cfg.tokenizer_user, cfg.tokenizer_pw)
  40.     filter_accessors_for_users = user_filter_factory(mongo.get_collection("user"))
  41.     accessors_anonymizer = AccessorsAnonymizer(tokenizer, filter_accessors_for_users)
  42.     bq = Bigquery(log, cfg.dataset_id, cfg.full_scan, cfg.execution_id, cfg.environment)
  43.  
  44.     def with_prefix(base_name: str):
  45.         return f"{cfg.table_prefix}_{base_name}"
  46.  
  47.     cmd_table = CommandTableFacade(log, bq, with_prefix("exec_commands"))
  48.     cmd_table.create()
  49.  
  50.     processors = [
  51.         Processor(
  52.             log,
  53.             bq,
  54.             cmd_table,
  55.             mongo.get_collection("installation"),
  56.             model.Table(
  57.                 with_prefix("exported_installations"),
  58.                 "",
  59.                 model.installation_table_definition(accessors_anonymizer),
  60.             ),
  61.             "installation",
  62.             mongo_cursor_batch_size=300,
  63.         ),
  64.         Processor(
  65.             log,
  66.             bq,
  67.             cmd_table,
  68.             mongo.get_collection("branch"),
  69.             model.Table(
  70.                 with_prefix("exported_branches"),
  71.                 "",
  72.                 model.branch_table_definition(),
  73.             ),
  74.             "branch",
  75.             mongo_cursor_batch_size=3000,
  76.         ),
  77.         Processor(
  78.             log,
  79.             bq,
  80.             cmd_table,
  81.             mongo.get_collection("space"),
  82.             model.Table(
  83.                 with_prefix("exported_spaces"),
  84.                 "",
  85.                 model.space_table_definition(accessors_anonymizer),
  86.             ),
  87.             "space",
  88.             mongo_cursor_batch_size=300,
  89.         ),
  90.         Processor(
  91.             log,
  92.             bq,
  93.             cmd_table,
  94.             mongo.get_collection("user"),
  95.             model.Table(
  96.                 with_prefix("exported_users"),
  97.                 "",
  98.                 model.user_table_definition(tokenizer),
  99.             ),
  100.             "user",
  101.             mongo_cursor_batch_size=1000,
  102.         ),
  103.     ]
  104.  
  105.     for p in processors:
  106.         p.setup()
  107.  
  108.     for p in processors:
  109.         p.export()
  110.  
  111.     log.info("Done")
Advertisement
Add Comment
Please, Sign In to add comment