Advertisement
Guest User

Untitled

a guest
Sep 15th, 2019
111
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 3.61 KB | None | 0 0
  1. import subprocess
  2. import sys
  3. import papermill as papermill
  4.  
  5.  
  6. REMOTE_FOLDER = "your cloud folder name"
  7. LOCAL_FOLDER = "your local folder name"
  8. TEMPLATE_NOTEBOOK = "template_notebook.ipynb"
  9.  
  10.  
  11. def get_new_files(remote_folder, local_folder):
  12. """
  13. A function that returns files that were uploaded to the cloud folder and do not exist in our local folder.
  14. """
  15. # list the files in our cloud folder
  16. list_cloud = subprocess.run(
  17. ["rclone", "lsf", f"remote:{remote_folder}"],
  18. capture_output=True,
  19. text=True,
  20. )
  21.  
  22. # transform the command output into a list
  23. cloud_directories = list_cloud.split("\n")[0:-1]
  24.  
  25. print(f"In the cloud we have: \n{cloud_directories}")
  26.  
  27. # list the files in our local folder
  28. list_cloud = subprocess.run(
  29. ["ls", local_folder], capture_output=True, text=True
  30. )
  31.  
  32. # transform the command output into a list
  33. local_directories = list_cloud.stdout.split("\n")[0:-1]
  34.  
  35. print(f"In the local copy we have: \n{local_directories}")
  36.  
  37. # create a list with the differences between the two lists above
  38. new_files = list(set(cloud_directories) - set(local_directories))
  39.  
  40. return new_files
  41.  
  42.  
  43. def sync_directories(remote_folder, local_folder):
  44. """
  45. A function that syncs a remote folder with a local folder
  46. with rclone.
  47. """
  48. sync = subprocess.run(
  49. ["rclone", "sync", f"remote:{remote_folder}", local_folder]
  50. )
  51.  
  52. print("Syncing local directory with cloud....")
  53. return sync.returncode
  54.  
  55.  
  56. def run_notebook(excel_report, template_notebook):
  57. """
  58. A function that runs a notebook against an excel report
  59. via papermill.
  60. """
  61. no_extension_name = excel_report.split(".")[0]
  62. papermill.execute_notebook(
  63. template_notebook,
  64. f"{no_extension_name}.ipynb",
  65. parameters=dict(filename=excel_report),
  66. )
  67. return no_extension_name
  68.  
  69.  
  70. def generate_html_report(notebook_file):
  71. """
  72. A function that converts a notebook into an html
  73. file.
  74. """
  75. generate = subprocess.run(
  76. ["jupyter", "nbconvert", notebook_file, "--to=html"]
  77. )
  78. print("HTML Report was generated")
  79. return True
  80.  
  81.  
  82. def push_to_cloud(remote_folder, filename):
  83. """
  84. A function that pushes to a remote cloud folder
  85. a specific file.
  86. """
  87.  
  88. push = subprocess.run(
  89. ["rclone", "copy", filename, f"remote:{remote_folder}"]
  90. )
  91. print("Report Published!!!")
  92.  
  93. def main():
  94. print("Starting updater..")
  95.  
  96. # detect if there are new files in the remote folder
  97. new_files = get_new_files(
  98. remote_folder=REMOTE_FOLDER, local_folder=LOCAL_FOLDER
  99. )
  100.  
  101. # if there are none, exit
  102. if not new_files:
  103. print("Everything is synced. No new files.")
  104. sys.exit()
  105. # else, continue
  106. else:
  107. print("There are files missing.")
  108. print(new_files)
  109.  
  110. # sync directories to get new excel report
  111. sync_directories(remote_folder=REMOTE_FOLDER, local_folder=LOCAL_FOLDER)
  112.  
  113. # generate new notebook and extract the name
  114. clean_name = run_notebook(new_files[0])
  115.  
  116. # the new notebook generate will have the following name
  117. notebook_name = f"{clean_name}.ipynb"
  118.  
  119. # generate the html report from the notebook
  120. generate_html_report(notebook_name)
  121.  
  122. # the notebook name will be the following
  123. html_report_name = f"{clean_name}.html"
  124.  
  125. # push the new notebook to the cloud
  126. push_to_cloud(html_report=html_report_name, remote_folder=ONEDRIVE_FOLDER)
  127.  
  128. # make sure everything is synced again
  129. sync_directories(remote_folder=REMOTE_FOLDER, local_folder=LOCAL_FOLDER)
  130.  
  131. print("Updater finished.")
  132.  
  133. return True
  134.  
  135.  
  136. if __name__ == "main":
  137. main()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement