Advertisement
Guest User

Untitled

a guest
Oct 19th, 2019
85
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 7.59 KB | None | 0 0
  1. # -*- coding:utf-8 -*-
  2.  
  3. """
  4. Python's duck typing system allows for very pleasant prototyping. But at some
  5. point, it is convenient to enforce some interfacing. This can be achieved
  6. Through abstract base classes and annotations to a great extent.
  7.  
  8. Another approach is to use dependency injection and inversion of control
  9. patterns. This gist exemplifies how to do that using ``dependency_injector``.
  10.  
  11. In this example, we have two types of dummy 'image processing' operations:
  12.  
  13. 1. pick one image and a range, and apply some nondeterministic transform
  14. 2. pick two images and two ranges, and fuse them
  15.  
  16. And we want to achieve the following goals
  17. 1. Automated way of ensuring that the op developers follow that interface
  18. 2. Flexible and efficient way of chaining and running the operations
  19. 3. Interface-agnostic interaction between operations and the file system
  20.  
  21. For that, we structure the code in 4 main sections:
  22. 1. Actual code which should fulfill given interfaces and have no side-effects
  23. 2. Definitions for the graph nodes, which enforce the interfaces
  24. 3. Definition of the computational graph
  25. 4. Main routine to run the graph
  26.  
  27. From these, only the first one provides functionality.
  28. * Abstracting point 2 allows to define different ways of interacting with
  29. the filesystem (or any other service) without impacting functionality.
  30. * Abstracting point 3 allows to define different chains of computations,
  31. while being parallelizable, thread-safe and efficient in terms of memory
  32. and runtime.
  33. * Abstracting point 4 allows to specify deployment conditions independently
  34. of the topology and semantics of the computation. Note how the graph container
  35. is responsible for creating, running and orchestrating the functional
  36. components. Also note how very dynamic configurations are still possible,
  37. despite the level of abstraction.
  38. """
  39.  
  40. import os
  41. import random
  42. from dependency_injector import containers, providers
  43.  
  44.  
  45. __author__ = "Andres FR"
  46.  
  47.  
  48. ################################################################################
  49. ### ORIGINAL OPERATIONS
  50. ################################################################################
  51.  
  52. def load_img_from_disk(img_dir, img_name):
  53. """
  54. A dummy image loader
  55. """
  56. return os.path.join(img_dir, img_name)
  57.  
  58. def img_operation1(img, a, b):
  59. """
  60. A dummy operation
  61. """
  62. num = random.randint(a, b)
  63. print("[img_operation1] called!", img, num)
  64. return img + " -> img_operation1_{}".format(num)
  65.  
  66.  
  67. def img_operation2(img, a, b):
  68. """
  69. A dummy operation
  70. """
  71. num = random.randint(a, b)
  72. print("[img_operation2] called!", img, num)
  73. return img + " -> img_operation2_{}".format(num)
  74.  
  75.  
  76. def img_operation3(img1, img2, a1, b1, a2, b2):
  77. """
  78. A dummy operation
  79. """
  80. bracket = [min(a1, a2), max(b1, b2)]
  81. print("[img_operation3] called!", img1, img2, bracket)
  82. return "img_operation3->{}".format(bracket)
  83.  
  84.  
  85. def main_routine(operation,
  86. img_names=("img_{}.png".format(x) for x in range(5))):
  87. """
  88. A loop of dummy operations
  89. """
  90. for i in img_names:
  91. result = operation(i)
  92. print(">>>>", result)
  93.  
  94.  
  95. ################################################################################
  96. ### "INTERFACE" DEFINITIONS
  97. ################################################################################
  98.  
  99. class ImageServer:
  100. """
  101. This mock object provides images and stores them in a cache.
  102. Its interface is the ``img_dir`` constructor parameter and the
  103. ``_call__(img_name) -> img`` method.
  104. """
  105.  
  106. def __init__(self, img_dir):
  107. """
  108. """
  109. print("[ImageServer] constructor called!")
  110. self.img_dir = img_dir
  111. self.images = {}
  112.  
  113. def __call__(self, name):
  114. """
  115. """
  116. try:
  117. img = self.images[name]
  118. print("[ImageServer] retrieving", name, "from cache")
  119. return img
  120. except KeyError:
  121. print("[ImageServer] loading", name, "from", self.img_dir)
  122. img = load_img_from_disk(self.img_dir, name)
  123. self.images[name] = img
  124. return img
  125.  
  126.  
  127. class BasicImgProcessingNode:
  128. """
  129. This mock object gets injected an image_server, and applies an operation
  130. to an image provided by it.
  131. The operations must fulfill the interface ``op(img, a, b)->img_result``
  132. """
  133. def __init__(self, img_server, operation, a, b):
  134. """
  135. """
  136. print("[BasicImgProcessingNode] constructor called with args:",
  137. operation.__name__, a, b)
  138. self.fn = operation
  139. self.a = a
  140. self.b = b
  141. self.img_server = img_server
  142.  
  143. def __call__(self, img_name):
  144. """
  145. """
  146. print("[BasicImgProcessingNode] called for image", img_name)
  147. img = self.img_server(img_name)
  148. output = self.fn(img, self.a, self.b)
  149. return output
  150.  
  151.  
  152. class FusionNode:
  153. """
  154. This mock object gets injected two image processing nodes. When called,
  155. applies them to an image and then passes the results to a given fusion
  156. operation, which fulfills the interface
  157. ``op(img1, img2 a1, b1, a2, b2)->img_result``
  158. """
  159.  
  160. def __init__(self, node1, node2, operation):
  161. """
  162. """
  163. print("[FusionNode] constructor called")
  164. self.n1 = node1
  165. self.n2 = node2
  166. self.fn = operation
  167.  
  168. def __call__(self, img_name):
  169. """
  170. """
  171. print("[FusionNode] called for image", img_name)
  172. im1 = self.n1(img_name)
  173. im2 = self.n2(img_name)
  174. output = self.fn(im1, im2, self.n1.a, self.n1.b, self.n2.a, self.n2.b)
  175. return output
  176.  
  177.  
  178. ################################################################################
  179. ### DEPENDENCY GRAPH
  180. ################################################################################
  181.  
  182.  
  183. class ComputationContainer(containers.DeclarativeContainer):
  184. """
  185. Once we took the job of defining the interfaces, putting the pieces together
  186. is straightforward. Note that this is highly customizable:
  187.  
  188. * The class can be extended, modified... to generate different comp. graphs
  189. * Config can be set at graph construction and overriden at any time after
  190. """
  191.  
  192. my_config = providers.Configuration("conf")
  193. img_server = providers.Singleton(ImageServer, my_config.img_dir)
  194. #
  195. process1 = providers.Factory(BasicImgProcessingNode, img_server,
  196. img_operation1, my_config.p1.a, my_config.p1.b)
  197. process2 = providers.Factory(BasicImgProcessingNode, img_server,
  198. img_operation2, my_config.p2.a, my_config.p2.b)
  199. fusion = providers.Factory(FusionNode, process1, process2, img_operation3)
  200. main = providers.Callable(main_routine, operation=fusion)
  201.  
  202.  
  203. ################################################################################
  204. ### MAIN ROUTINE
  205. ################################################################################
  206.  
  207. if __name__ == "__main__":
  208.  
  209. # construct the graph
  210. CONTAINER = ComputationContainer(
  211. my_config={
  212. "img_dir": "/test/images",
  213. "p1": {"a": 0, "b": 1000},
  214. "p2": {"a": 1001, "b": 2000}
  215. }
  216. )
  217.  
  218. print("\n\nTEST CONFIG OVERRIDE AND RETRIEVING IMAGE FROM CACHE:\n\n")
  219.  
  220. p1_low = CONTAINER.process1()
  221. CONTAINER.my_config.override({"p1": {"a": 100000, "b": 2000000}})
  222. p1_high = CONTAINER.process1()
  223. print("\n")
  224. p1_low("hello1")
  225. print("\n")
  226. p1_low("hello2")
  227. print("\n")
  228. p1_high("hello1")
  229.  
  230. print("\n\nTEST FUSION NODE AND RETRIEVING IMAGE FROM CACHE:\n\n")
  231.  
  232. p_fusion = CONTAINER.fusion()
  233. print("\n")
  234. p_fusion("hello_fusion")
  235. print("\n")
  236. p_fusion("hello_fusion")
  237.  
  238. print("\n\nRUN MAIN ROUTINE:\n\n")
  239.  
  240. CONTAINER.main()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement