API tools faq
paste
Advertisement
Guest User

Untitled

a guest
Apr 26th, 2019
83
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 9.51 KB | None | 0 0
raw download clone embed print report
  1. {
  2. "cells": [
  3. {
  4. "cell_type": "code",
  5. "execution_count": 73,
  6. "metadata": {},
  7. "outputs": [],
  8. "source": [
  9. " import nbformat, time, jsonpatch, jsonpointer, itertools, jsondiff, json, logging\n",
  10. " from IPython import get_ipython\n",
  11. " import datetime, notebook, traitlets, pathlib, jsonpatch, pathlib, mimetypes\n",
  12. " import ipywidgets, nbconvert, IPython, html"
  13. ]
  14. },
  15. {
  16. "cell_type": "code",
  17. "execution_count": 74,
  18. "metadata": {},
  19. "outputs": [],
  20. "source": [
  21. " def strip_remove_patch_value(patches):\n",
  22. " [(patch['op'] == 'remove') and patch.pop('value', None) for patch in patches]"
  23. ]
  24. },
  25. {
  26. "cell_type": "code",
  27. "execution_count": 75,
  28. "metadata": {},
  29. "outputs": [],
  30. "source": [
  31. " cache = {}\n",
  32. " last_patch = []"
  33. ]
  34. },
  35. {
  36. "cell_type": "code",
  37. "execution_count": 76,
  38. "metadata": {},
  39. "outputs": [],
  40. "source": [
  41. " def remove_trust(nb):\n",
  42. " for cell in nb['cells']: cell['metadata'].pop('trusted', None)"
  43. ]
  44. },
  45. {
  46. "cell_type": "code",
  47. "execution_count": 77,
  48. "metadata": {},
  49. "outputs": [],
  50. "source": [
  51. " def source2lines(nb):\n",
  52. " for cell in nb['cells']:\n",
  53. " key = 'text' if 'text' in cell else 'source'\n",
  54. " if isinstance(cell[key], str):\n",
  55. " cell[key] = cell[key].splitlines(True)"
  56. ]
  57. },
  58. {
  59. "cell_type": "code",
  60. "execution_count": 78,
  61. "metadata": {},
  62. "outputs": [],
  63. "source": [
  64. " def write_resources(source, target, parent, *patches):\n",
  65. " t = target['metadata']['last_modified']\n",
  66. " for i, (src, tgt) in enumerate(\n",
  67. " itertools.zip_longest((source or target)['cells'], target['cells'])\n",
  68. " ):\n",
  69. " orig, new = (src or {}).get('outputs', []), (tgt or {}).get('outputs', [])\n",
  70. " if orig == new and not (orig is new): continue\n",
  71. " \n",
  72. " for j, (p, n) in enumerate(itertools.zip_longest(orig, new)):\n",
  73. " if p == n and not (p is n): continue\n",
  74. " if (n or {}).get('output_type', None) not in {'execute_result', 'display_data'}: \n",
  75. " continue\n",
  76. " for type, value in (n or {}).get('data', {}).items():\n",
  77. " if type == 'text/plain': continue\n",
  78. " resource_path = F\"{t}_{i}_{j}{mimetypes.guess_extension(type)}\"\n",
  79. " path = parent / jsonpointer.escape(type) / resource_path\n",
  80. " path.parent.mkdir(parents=True, exist_ok=True)\n",
  81. " if isinstance(value, str): path.write_text(value)\n",
  82. " elif isinstance(value, bytes): path.write_bytes(value)\n",
  83. " patches += {\n",
  84. " \"op\": \"replace\", \"value\": str(path), \"path\": F\"/cells/{i}/outputs/{j}/data/{jsonpointer.escape(type)}\"\n",
  85. " },\n",
  86. " \n",
  87. " return patches"
  88. ]
  89. },
  90. {
  91. "cell_type": "code",
  92. "execution_count": 79,
  93. "metadata": {},
  94. "outputs": [],
  95. "source": [
  96. " def pre_save_hook(model=None, path=None, contents_manager=None):\n",
  97. " \n",
  98. " if model[\"type\"] == \"notebook\":\n",
  99. " model = {**contents_manager.get(path), 'content': model['content']}\n",
  100. " global cache, last_patch\n",
  101. "\n",
  102. " path = pathlib.Path(path)\n",
  103. " nb = model['content']\n",
  104. " \n",
  105. " \n",
  106. " copy = nbformat.from_dict(nb)\n",
  107. " source2lines(copy)\n",
  108. " \n",
  109. " remove_trust(copy)\n",
  110. " copy['metadata'].update(last_modified=str(int(model[\"last_modified\"].timestamp())))\n",
  111. " cache[path] = cache.get(path, nbformat.v4.new_notebook())\n",
  112. " resource_path = path.parent / '.nbstream' / path.stem\n",
  113. " copy = nbformat.from_dict(copy)\n",
  114. " files_patch = write_resources(cache[path], copy, resource_path)\n",
  115. " \n",
  116. " change = jsonpatch.JsonPatch.from_diff(\n",
  117. " jsonpatch.apply_patch(copy, files_patch), \n",
  118. " jsonpatch.apply_patch(cache[path], last_patch)\n",
  119. " )\n",
  120. " \n",
  121. " if len(change.patch) == 1 and change.patch[0]['path'] == '/metadata/last_modified': return\n",
  122. "\n",
  123. " log_path = path.parent / '.nbstream' / path.with_suffix(\".json\").name\n",
  124. " if not log_path.exists():\n",
  125. " log_path.parent.mkdir(parents=True, exist_ok=True)\n",
  126. " log_path.touch()\n",
  127. "\n",
  128. " if change.patch:\n",
  129. " with open(log_path, \"a\") as file:\n",
  130. " file.write(\"\".join(json.dumps(change.patch).splitlines()))\n",
  131. " file.write(\"\\n\")\n",
  132. " cache[path], last_patch = copy, files_patch"
  133. ]
  134. },
  135. {
  136. "cell_type": "code",
  137. "execution_count": 80,
  138. "metadata": {},
  139. "outputs": [],
  140. "source": [
  141. " def join_source(nb):\n",
  142. " for cell in nb['cells']:\n",
  143. " if 'source' in cell:\n",
  144. " cell['source'] = ''.join(cell['source'])"
  145. ]
  146. },
  147. {
  148. "cell_type": "code",
  149. "execution_count": 85,
  150. "metadata": {},
  151. "outputs": [],
  152. "source": [
  153. " def time_masheen(name):\n",
  154. " changes = list(map(json.loads, pathlib.Path(F'.nbstream/{name}.json').read_text().splitlines()))\n",
  155. "\n",
  156. " NB = json.loads(pathlib.Path(F'.ipynb_checkpoints/{name}-checkpoint.ipynb').read_text())\n",
  157. "\n",
  158. " @ipywidgets.interact\n",
  159. " def _(i=(0, len(changes)), e=nbconvert.get_export_names()):\n",
  160. " nonlocal NB\n",
  161. " nb = nbformat.from_dict(NB)\n",
  162. " for change in reversed(changes[-i:]):\n",
  163. " nb = jsonpatch.apply_patch(nb, change)\n",
  164. " join_source(nb)\n",
  165. " try:\n",
  166. " %pushd ~\n",
  167. " resolve_output_files(nb)\n",
  168. " finally:\n",
  169. " %popd\n",
  170. " object = nbconvert.get_exporter(e)().from_notebook_node(nbformat.from_dict(nb))[0]\n",
  171. "\n",
  172. " if e.startswith('html'):\n",
  173. " object = IPython.display.HTML(\n",
  174. " F\"\"\"<iframe srcdoc=\"{html.escape(object)}\" width=\"100%\" height=\"600\"></iframe>\"\"\"\n",
  175. " )\n",
  176. " if e in {'python', 'script'}: object = IPython.display.Code(object)\n",
  177. "\n",
  178. " if e in {'markdown'}: object = IPython.display.Markdown(object)\n",
  179. " \n",
  180. " if e in {'notebook', 'json'}: object = IPython.display.JSON(object)\n",
  181. "\n",
  182. " IPython.display.display(object)\n",
  183. " return _"
  184. ]
  185. },
  186. {
  187. "cell_type": "code",
  188. "execution_count": 86,
  189. "metadata": {},
  190. "outputs": [],
  191. "source": [
  192. " def resolve_output_files(nb):\n",
  193. " for cell in nb['cells']:\n",
  194. " for output in cell.get('outputs', []):\n",
  195. " for type, value in output.get('data', {}).items():\n",
  196. " try:\n",
  197. " if isinstance(value, str) and pathlib.Path(value).exists():\n",
  198. " output['data'][type] = pathlib.Path(value).read_text()\n",
  199. " except OSError: ..."
  200. ]
  201. },
  202. {
  203. "cell_type": "code",
  204. "execution_count": 87,
  205. "metadata": {},
  206. "outputs": [],
  207. "source": [
  208. "# c = cm.get('2019-04-15-Untitled.ipynb', True)\n",
  209. "\n",
  210. "# jsonpatch.apply_patch(\n",
  211. "# c['content'],\n",
  212. "# write_resources(nbformat.v4.new_notebook(), c['content'], pathlib.Path('.nbstream/xxx'))\n",
  213. "# )"
  214. ]
  215. },
  216. {
  217. "cell_type": "code",
  218. "execution_count": null,
  219. "metadata": {},
  220. "outputs": [],
  221. "source": [
  222. " if __name__ == '__main__':\n",
  223. " !jupyter nbconvert --to python napp.ipynb\n",
  224. " !black napp.py\n",
  225. " !isort napp.py"
  226. ]
  227. },
  228. {
  229. "cell_type": "code",
  230. "execution_count": null,
  231. "metadata": {},
  232. "outputs": [],
  233. "source": []
  234. },
  235. {
  236. "cell_type": "code",
  237. "execution_count": null,
  238. "metadata": {},
  239. "outputs": [],
  240. "source": []
  241. }
  242. ],
  243. "metadata": {
  244. "kernelspec": {
  245. "display_name": "p6",
  246. "language": "python",
  247. "name": "other-env"
  248. },
  249. "language_info": {
  250. "codemirror_mode": {
  251. "name": "ipython",
  252. "version": 3
  253. },
  254. "file_extension": ".py",
  255. "mimetype": "text/x-python",
  256. "name": "python",
  257. "nbconvert_exporter": "python",
  258. "pygments_lexer": "ipython3",
  259. "version": "3.6.8"
  260. },
  261. "last_modified": "1555354753",
  262. "varInspector": {
  263. "cols": {
  264. "lenName": 16,
  265. "lenType": 16,
  266. "lenVar": 40
  267. },
  268. "kernels_config": {
  269. "python": {
  270. "delete_cmd_postfix": "",
  271. "delete_cmd_prefix": "del ",
  272. "library": "var_list.py",
  273. "varRefreshCmd": "print(var_dic_list())"
  274. },
  275. "r": {
  276. "delete_cmd_postfix": ") ",
  277. "delete_cmd_prefix": "rm(",
  278. "library": "var_list.r",
  279. "varRefreshCmd": "cat(var_dic_list()) "
  280. }
  281. },
  282. "types_to_exclude": [
  283. "module",
  284. "function",
  285. "builtin_function_or_method",
  286. "instance",
  287. "_Feature"
  288. ],
  289. "window_display": false
  290. }
  291. },
  292. "nbformat": 4,
  293. "nbformat_minor": 4
  294. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!