Advertisement
Guest User

Untitled

a guest
Aug 5th, 2023
230
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 6.35 KB | None | 0 0
  1. import torch
  2. from torch.nn.functional import cosine_similarity as torch_cosine_similarity
  3. import torchvision.models as models
  4. from torchvision.models import ResNet18_Weights
  5. from torchvision import transforms
  6. from PIL import Image
  7. import math
  8. import cv2
  9. import random
  10.  
  11. torch.set_grad_enabled(False)
  12. torch.use_deterministic_algorithms(True)
  13. torch.manual_seed(0)
  14.  
  15.  
  16. def cossim(tensor1, tensor2):
  17. # Flatten the tensors
  18. flat_tensor1 = tensor1.flatten()
  19. flat_tensor2 = tensor2.flatten()
  20.  
  21. # Assert that the tensors have the same size
  22. assert flat_tensor1.shape == flat_tensor2.shape, "Tensors must have the same size"
  23.  
  24. # Calculate cosine similarity
  25. similarity = torch_cosine_similarity(flat_tensor1.unsqueeze(0), flat_tensor2.unsqueeze(0), dim=-1)
  26.  
  27. return similarity.item()
  28.  
  29. def cossim_in_degrees(a, b):
  30. return math.degrees(math.acos(cossim(a,b)))
  31.  
  32. def nudge(tensor, nudge_magnitude=1e-5, use_random_gaussian=True):
  33. original_magnitude = torch.norm(tensor)
  34.  
  35. if use_random_gaussian:
  36. # Create a tensor with the same shape as the input tensor, filled with random Gaussian values
  37. nudge_tensor = torch.randn_like(tensor)
  38. else:
  39. # Create a tensor of ones with the same shape as the input tensor
  40. nudge_tensor = torch.ones_like(tensor)
  41.  
  42. scaled_nudge_tensor = nudge_tensor * (original_magnitude * nudge_magnitude)
  43. modified_tensor = tensor + scaled_nudge_tensor
  44. new_magnitude = torch.norm(modified_tensor)
  45. scaling_factor = original_magnitude / new_magnitude
  46. final_tensor = modified_tensor * scaling_factor
  47. return final_tensor
  48.  
  49.  
  50. def get_distortion_ratio(point, black_box):
  51. nudged = nudge(point)
  52. control = cossim_in_degrees(point, nudged)
  53. mapped_point = black_box(point)
  54. mapped_nudged = black_box(nudged)
  55. test = cossim_in_degrees(mapped_point, mapped_nudged)
  56. return test / control
  57.  
  58. def get_conv_outputs(input_tensor):
  59. # List to store the outputs of all convolutional layers
  60. conv_outputs = []
  61.  
  62. # Hook function to store the output of a convolutional layer
  63. def hook(module, input, output):
  64. if isinstance(module, torch.nn.Conv2d):
  65. conv_outputs.append(output)
  66.  
  67. # List to store handles for the hooks
  68. handles = []
  69.  
  70. # Register the hook to all modules within the model and store handles
  71. for module in resnet18.modules():
  72. handle = module.register_forward_hook(hook)
  73. handles.append(handle)
  74.  
  75. # Running the forward pass
  76. with torch.no_grad():
  77. resnet18(input_tensor)
  78.  
  79. # Unregistering the hooks
  80. for handle in handles:
  81. handle.remove()
  82.  
  83. return conv_outputs
  84.  
  85.  
  86. def image_path_to_resnet18_input(image_path):
  87. # Define the preprocessing pipeline
  88. preprocess = transforms.Compose([
  89. transforms.Resize(256),
  90. transforms.CenterCrop(224),
  91. transforms.ToTensor(),
  92. transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
  93. ])
  94.  
  95. # Load the image from the provided path
  96. image = Image.open(image_path)
  97.  
  98. # Convert the image to RGB if it has an alpha channel
  99. if image.mode == 'RGBA':
  100. image = image.convert('RGB')
  101.  
  102. # Preprocess the image
  103. input_tensor = preprocess(image)
  104.  
  105. # Add a batch dimension (since PyTorch expects a batch)
  106. input_tensor = input_tensor.unsqueeze(0)
  107.  
  108. return input_tensor
  109.  
  110. def generalized_outer_product(vector_array):
  111. alphabet = "ijklmnopqrstuvwxyzabcdefgh"
  112. einsum_delimiter = ", "
  113. einsum_args = ""
  114. einsum_result = ""
  115. for i in range(len(vector_array)):
  116. einsum_args += alphabet[i]
  117. if i != len(vector_array) - 1:
  118. einsum_args += einsum_delimiter
  119. einsum_result += alphabet[i]
  120. einsum_equation = einsum_args + " -> " + einsum_result
  121. return torch.einsum(einsum_equation, vector_array)
  122.  
  123. # expects an array of matrices of shape (r,*) (an array of vectors of any size is okay too, it implies r=1)
  124. def unfurl_rank_r_tensor(matrix_array):
  125. def conditional_unsqueeze(suspected_vec):
  126. if len(suspected_vec.size()) < 2:
  127. return torch.unsqueeze_copy(suspected_vec, 0)
  128. return suspected_vec
  129. if len(matrix_array) == 0:
  130. return torch.tensor([])
  131. rank = conditional_unsqueeze(matrix_array[0]).size()[0]
  132. for matrix in matrix_array:
  133. assert matrix.size()[0] == rank
  134. target_shape = [m.size()[1] for m in matrix_array]
  135. unfurled_tensor = torch.zeros(target_shape)
  136. for i in range(rank):
  137. outer_product_vector_array = [m[i] for m in matrix_array]
  138. unfurled_tensor += generalized_outer_product(outer_product_vector_array)
  139. return unfurled_tensor
  140.  
  141. def frame_to_resnet18_input(frame):
  142. # Convert the frame to a PIL Image
  143. image = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
  144.  
  145. # Preprocess the image
  146. preprocess = transforms.Compose([
  147. transforms.Resize(256),
  148. transforms.CenterCrop(224),
  149. transforms.ToTensor(),
  150. transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
  151. ])
  152.  
  153. input_tensor = preprocess(image)
  154. input_tensor = input_tensor.unsqueeze(0)
  155. return input_tensor
  156.  
  157.  
  158. resnet18 = models.resnet18(weights=ResNet18_Weights.IMAGENET1K_V1)
  159.  
  160. # the black box is assumed to map flat tensors to flat tesnors of the same length. that's it.
  161. def black_box(x):
  162. y = get_conv_outputs(x)[-3]
  163. return y
  164.  
  165.  
  166. paths = ['pfp.png', 'pfp-m1.png', 'pfp-m2.png', 'pfp-m3.png', 'lizard.jpeg']
  167.  
  168. # path = paths[2]
  169.  
  170. for i in range(len(paths)):
  171. x = image_path_to_resnet18_input(paths[i])
  172. r = get_distortion_ratio(x, black_box)
  173. print(r)
  174. print('\n\n')
  175.  
  176.  
  177.  
  178.  
  179. '''
  180.  
  181. # Open a connection to the webcam
  182. cap = cv2.VideoCapture(0)
  183.  
  184. while True:
  185. # Capture frame-by-frame
  186. ret, frame = cap.read()
  187.  
  188. # Preprocess the frame
  189. input_tensor = frame_to_resnet18_input(frame)
  190.  
  191. # Get the number from the CNN
  192. number = get_distortion_ratio(input_tensor, black_box)
  193.  
  194. # Draw the number on the frame
  195. cv2.putText(frame, str(number), (10, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2, cv2.LINE_AA)
  196.  
  197. # Display the frame
  198. cv2.imshow('Frame', frame)
  199. if cv2.waitKey(1) & 0xFF == ord('q'):
  200. break
  201.  
  202. # Release the capture when done
  203. cap.release()
  204. cv2.destroyAllWindows()
  205.  
  206. '''
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement