Guest User

Untitled

a guest
Dec 2nd, 2024
77
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 19.33 KB | None | 0 0
  1. import time
  2. import torch
  3. from PIL import Image, PngImagePlugin
  4. import os
  5. import numpy as np
  6. from skimage.metrics import structural_similarity as ssim
  7. import cv2
  8. import math
  9. from typing import List
  10.  
  11. # Constants
  12. DEFAULT_MIN_FRAMES_PER_SLICE = 10 # Minimum frames to extract per slice
  13. DEFAULT_MAX_FRAMES_PER_SLICE = 100 # Maximum frames to extract per slice
  14. DEFAULT_FPS_PER_SLICE = 10 # Frames per second to extract per slice
  15. SIMILARITY_THRESHOLD = 0.5
  16. DEFAULT_WIDTH = 1024
  17. DEFAULT_HEIGHT = 1024
  18.  
  19. UPSCALE_MODEL_NAME = "caidas/swin2SR-lightweight-x2-64"
  20.  
  21. # Global variables for models and device
  22. global_processor = None
  23. global_model = None
  24. global_device = None
  25.  
  26. def load_models():
  27. """
  28. Loads the Swin2SR model and processor.
  29. """
  30. global global_processor
  31. global global_model
  32. global global_device
  33.  
  34. from transformers import Swin2SRImageProcessor, Swin2SRForImageSuperResolution
  35. print("Loading models...")
  36. model_name = UPSCALE_MODEL_NAME
  37. global_processor = Swin2SRImageProcessor.from_pretrained(model_name)
  38. global_model = Swin2SRForImageSuperResolution.from_pretrained(model_name)
  39. global_model.eval()
  40.  
  41. global_device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
  42. global_model.to(global_device)
  43. print("Models loaded.")
  44.  
  45. def unload_models():
  46. """
  47. Unloads the models and clears GPU memory.
  48. """
  49. global global_processor
  50. global global_model
  51. global global_device
  52. print("Unloading models...")
  53. if global_model is not None:
  54. del global_model
  55. if global_processor is not None:
  56. del global_processor
  57. torch.cuda.empty_cache()
  58. print("Models unloaded.")
  59.  
  60. def extract_frames_from_video(input_video, output_dir, original_file_name, min_frames_per_slice, max_frames_per_slice,
  61. fps=DEFAULT_FPS_PER_SLICE, start_point=0.0, end_point=None, video_slice_count=1):
  62. """
  63. Extract frames from video with control over frame extraction rate, slicing, and a limit on total images.
  64. Returns a list of file paths to the saved frames.
  65. """
  66. cap = cv2.VideoCapture(input_video)
  67. video_fps = cap.get(cv2.CAP_PROP_FPS)
  68. if video_fps == 0:
  69. video_fps = fps # Default to requested fps if video fps is unavailable
  70. video_frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
  71. video_duration = video_frame_count / video_fps # in seconds
  72.  
  73. # Set default end_point if not provided
  74. if end_point is None or end_point > video_duration:
  75. end_point = video_duration
  76.  
  77. # Use floor and ceil for start and end points
  78. start_point = max(0.0, math.floor(start_point))
  79. end_point = min(video_duration, math.ceil(end_point))
  80.  
  81. # Calculate slice durations
  82. total_duration = end_point - start_point
  83. if video_slice_count != 0:
  84. slice_duration = total_duration / video_slice_count
  85. else:
  86. slice_duration = total_duration
  87.  
  88. saved_frame_paths = []
  89.  
  90. for slice_idx in range(video_slice_count):
  91. slice_start = start_point + slice_idx * slice_duration
  92. slice_end = min(start_point + (slice_idx + 1) * slice_duration, end_point)
  93.  
  94. # Set the video to the start of the slice
  95. cap.set(cv2.CAP_PROP_POS_MSEC, slice_start * 1000)
  96.  
  97. frames = []
  98. extracted_frames = 0
  99. frame_count = 0
  100.  
  101. while cap.isOpened():
  102. current_time = cap.get(cv2.CAP_PROP_POS_MSEC) / 1000 # Current time in seconds
  103.  
  104. if current_time > slice_end:
  105. break
  106.  
  107. ret, frame = cap.read()
  108. if not ret:
  109. break
  110.  
  111. if frame_count % max(int(video_fps / fps), 1) == 0:
  112. # Convert frame to RGB
  113. frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
  114. frames.append((frame_rgb, current_time))
  115. extracted_frames += 1
  116.  
  117. frame_count += 1
  118.  
  119. if frames:
  120. # Adjust frames to meet min and max frame requirements
  121. frames = adjust_frame_count(frames, min_frames_per_slice, max_frames_per_slice, SIMILARITY_THRESHOLD)
  122.  
  123. # Save frames with metadata
  124. frame_paths = save_frames(frames, output_dir, original_file_name, slice_idx, start_point=slice_start)
  125. saved_frame_paths.extend(frame_paths)
  126.  
  127. cap.release()
  128. print(f"Extracted and saved frames from video.")
  129. return saved_frame_paths
  130.  
  131. def extract_frames_from_gif(input_gif, output_dir, original_file_name, min_frames_per_slice, max_frames_per_slice,
  132. fps=DEFAULT_FPS_PER_SLICE, start_point=0.0, end_point=None, gif_slice_count=1):
  133. """
  134. Extract frames from GIF with control over frame extraction rate, slicing, and a limit on total images.
  135. Returns a list of file paths to the saved frames.
  136. """
  137. gif = Image.open(input_gif)
  138. gif_duration = gif.info.get('duration', 100) # Duration of each frame in milliseconds
  139. total_frames = gif.n_frames
  140. total_duration = (gif_duration * total_frames) / 1000.0 # in seconds
  141.  
  142. # Set default end_point if not provided
  143. if end_point is None or end_point > total_duration:
  144. end_point = total_duration
  145.  
  146. # Use floor and ceil for start and end points
  147. start_point = max(0.0, math.floor(start_point))
  148. end_point = min(total_duration, math.ceil(end_point))
  149.  
  150. # Calculate slice durations
  151. total_duration = end_point - start_point
  152. slice_duration = total_duration / gif_slice_count
  153.  
  154. saved_frame_paths = []
  155.  
  156. for slice_idx in range(gif_slice_count):
  157. slice_start = start_point + slice_idx * slice_duration
  158. slice_end = min(start_point + (slice_idx + 1) * slice_duration, end_point)
  159.  
  160. frames = []
  161. extracted_frames = 0
  162. frame_index = 0
  163. current_time = 0.0
  164.  
  165. try:
  166. gif.seek(0)
  167. while True:
  168. current_time = (gif.tell() * gif_duration) / 1000.0 # Current time in seconds
  169.  
  170. if current_time >= slice_start and current_time <= slice_end:
  171. frame = gif.convert('RGB')
  172. frames.append((np.array(frame), current_time))
  173. extracted_frames += 1
  174. else:
  175. if current_time > slice_end:
  176. break
  177.  
  178. gif.seek(gif.tell() + 1)
  179. frame_index += 1
  180. except EOFError:
  181. pass
  182.  
  183. if frames:
  184. # Adjust frames to meet min and max frame requirements
  185. frames = adjust_frame_count(frames, min_frames_per_slice, max_frames_per_slice, SIMILARITY_THRESHOLD)
  186.  
  187. # Save frames with metadata
  188. frame_paths = save_frames(frames, output_dir, original_file_name, slice_idx, start_point=slice_start)
  189. saved_frame_paths.extend(frame_paths)
  190.  
  191. print(f"Extracted and saved frames from GIF.")
  192. return saved_frame_paths
  193.  
  194. def adjust_frame_count(frames_with_times, min_frames, max_frames, similarity_threshold):
  195. """
  196. Adjusts the number of frames to be within min_frames and max_frames by interpolating or reducing frames.
  197. """
  198. current_total_frames = len(frames_with_times)
  199.  
  200. if current_total_frames < min_frames:
  201. frames_with_times = interpolate_frames(frames_with_times, min_frames - current_total_frames, similarity_threshold)
  202. elif current_total_frames > max_frames:
  203. frames_with_times = reduce_frames(frames_with_times, max_frames)
  204.  
  205. return frames_with_times
  206.  
  207. def interpolate_frames(frames_with_times, frames_needed, similarity_threshold):
  208. """
  209. Interpolates frames to increase the total number of frames.
  210. """
  211. print(f"Interpolating {frames_needed} frames to meet minimum frame requirement.")
  212. interpolation_candidates = []
  213. for i in range(len(frames_with_times) - 1):
  214. current_frame, _ = frames_with_times[i]
  215. next_frame, _ = frames_with_times[i + 1]
  216.  
  217. # Convert to grayscale for SSIM comparison
  218. gray_current = cv2.cvtColor(current_frame, cv2.COLOR_RGB2GRAY)
  219. gray_next = cv2.cvtColor(next_frame, cv2.COLOR_RGB2GRAY)
  220.  
  221. # Calculate SSIM (Structural Similarity Index)
  222. score = ssim(gray_current, gray_next)
  223.  
  224. # Interpolate if the frames are similar enough
  225. if score >= similarity_threshold:
  226. interpolation_candidates.append(i)
  227.  
  228. if not interpolation_candidates:
  229. print("No suitable frames for interpolation.")
  230. return frames_with_times
  231.  
  232. num_interpolations = min(len(interpolation_candidates), frames_needed)
  233. indices_to_interpolate = np.linspace(0, len(interpolation_candidates) - 1, num_interpolations, dtype=int)
  234. indices_to_interpolate = [interpolation_candidates[i] for i in indices_to_interpolate]
  235.  
  236. interpolated = []
  237. for i in range(len(frames_with_times)):
  238. interpolated.append(frames_with_times[i])
  239. if i in indices_to_interpolate and i + 1 < len(frames_with_times):
  240. current_frame, current_time = frames_with_times[i]
  241. next_frame, next_time = frames_with_times[i + 1]
  242. blended_frame = np.uint8((current_frame.astype(np.float32) + next_frame.astype(np.float32)) / 2)
  243. blended_time = (current_time + next_time) / 2
  244. interpolated.append((blended_frame, blended_time))
  245.  
  246. return interpolated
  247.  
  248. def reduce_frames(frames_with_times, max_frames):
  249. """
  250. Reduces the number of frames to meet the maximum frame limit.
  251. """
  252. print(f"Reducing frames to meet maximum frame limit of {max_frames}.")
  253. total_frames = len(frames_with_times)
  254. indices_to_keep = np.linspace(0, total_frames - 1, max_frames, dtype=int)
  255. reduced_frames = [frames_with_times[i] for i in indices_to_keep]
  256. return reduced_frames
  257.  
  258. def save_frames(frames_with_times, output_dir, original_file_name, slice_number, start_point=0.0):
  259. """
  260. Saves frames to the output directory with sequential numerical filenames and metadata.
  261. Returns a list of file paths to the saved frames.
  262. """
  263. saved_frame_paths = []
  264. if not os.path.exists(output_dir):
  265. os.makedirs(output_dir)
  266.  
  267. for idx, (frame, frame_time) in enumerate(frames_with_times):
  268. img = Image.fromarray(frame)
  269. frame_number = idx
  270. frame_filename = f"{original_file_name}_slice_{slice_number}_frame_{frame_number}.png"
  271. frame_path = os.path.join(output_dir, frame_filename)
  272.  
  273. # Prepare metadata
  274. metadata = PngImagePlugin.PngInfo()
  275. metadata.add_text("Original Media", original_file_name)
  276. metadata.add_text("Slice Number", str(slice_number))
  277. metadata.add_text("Frame Count", str(len(frames_with_times)))
  278. metadata.add_text("Frame Time (ms)", str(int(frame_time * 1000)))
  279. if idx % 2 == 1:
  280. # This frame is interpolated
  281. metadata.add_text("Interpolated", "True")
  282. previous_time = frames_with_times[idx - 1][1]
  283. time_difference = frame_time - previous_time
  284. metadata.add_text("Time Difference (ms)", str(int(time_difference * 1000)))
  285. else:
  286. metadata.add_text("Interpolated", "False")
  287.  
  288. img.save(frame_path, pnginfo=metadata)
  289. saved_frame_paths.append(frame_path)
  290.  
  291. return saved_frame_paths
  292.  
  293. def resize_with_aspect_ratio(img, min_size, max_size):
  294. """
  295. Resizes an image to fit within min_size and max_size while maintaining aspect ratio.
  296. """
  297. width, height = img.size
  298. aspect_ratio = width / height
  299.  
  300. # Calculate scaling factors
  301. width_scale_min = min_size[0] / width
  302. height_scale_min = min_size[1] / height
  303. width_scale_max = max_size[0] / width
  304. height_scale_max = max_size[1] / height
  305.  
  306. # Determine the scaling factor for upscaling or downscaling
  307. scale_factor = 1.0
  308.  
  309. # Upscale if necessary
  310. if width < min_size[0] or height < min_size[1]:
  311. scale_factor = max(width_scale_min, height_scale_min)
  312. # Downscale if necessary
  313. if width > max_size[0] or height > max_size[1]:
  314. scale_factor = min(width_scale_max, height_scale_max)
  315.  
  316. # Apply scaling if needed
  317. if scale_factor != 1.0:
  318. new_width = int(width * scale_factor)
  319. new_height = int(height * scale_factor)
  320. img = img.resize((new_width, new_height), resample=Image.Resampling.LANCZOS)
  321.  
  322. return img
  323.  
  324. def adjust_frame_sizes(output_dir, min_size, max_size, processor, model, device):
  325. """
  326. Adjusts the size of all frames in the output directory to fit within min_size and max_size while maintaining aspect ratio.
  327. Returns a list of file paths to the resized images.
  328. """
  329. print("Adjusting frame sizes...")
  330.  
  331. resized_frame_paths = []
  332.  
  333. for filename in os.listdir(output_dir):
  334. if filename.endswith('.png'):
  335. img_path = os.path.join(output_dir, filename)
  336. img = Image.open(img_path).convert('RGB')
  337. width, height = img.size
  338.  
  339. # Determine if upscaling or downscaling is needed
  340. need_upscale = width < min_size[0] or height < min_size[1]
  341. need_downscale = width > max_size[0] or height > max_size[1]
  342.  
  343. # Upscale if necessary
  344. if need_upscale:
  345. img = upscale_image(img, model, processor, device)
  346. print(f"Upscaled {filename} to meet minimum size {min_size}.")
  347.  
  348. # After upscaling, update width and height
  349. width, height = img.size
  350.  
  351. # Resize to fit within min_size and max_size while preserving aspect ratio
  352. img = resize_with_aspect_ratio(img, min_size, max_size)
  353.  
  354. # Save the processed image back
  355. img.save(img_path)
  356. resized_frame_paths.append(img_path)
  357.  
  358. print(f"Size adjustment complete. Frames saved in {output_dir}.")
  359. return resized_frame_paths
  360.  
  361. def upscale_image(img, model, processor, device):
  362. """
  363. Upscales an image using the specified Hugging Face model.
  364. """
  365. # Preprocess the image
  366. inputs = processor(images=img, return_tensors="pt").to(device)
  367.  
  368. # Perform inference
  369. with torch.no_grad():
  370. outputs = model(**inputs)
  371.  
  372. # Post-process the output
  373. output = outputs.reconstruction.data.squeeze().clamp(0, 1).cpu().numpy()
  374. output = (output * 255.0).astype(np.uint8)
  375. output = np.transpose(output, (1, 2, 0))
  376. upscaled_img = Image.fromarray(output)
  377.  
  378. return upscaled_img
  379.  
  380. def process_media(
  381. input_media,
  382. output_base_dir,
  383. min_frames_per_slice,
  384. max_frames_per_slice,
  385. fps,
  386. similarity_threshold,
  387. min_size,
  388. max_size,
  389. start_point=0.0,
  390. end_point=None,
  391. slice_count=1
  392. ):
  393. """
  394. Processes a media file (video or GIF) and returns a list of paths to the created images.
  395. """
  396. original_file_name = os.path.splitext(os.path.basename(input_media))[0]
  397.  
  398. # Create a unique output directory per input media file
  399. timestamp = time.strftime("%Y%m%d-%H%M%S")
  400. output_dir = os.path.join(output_base_dir, f"{original_file_name}_{timestamp}")
  401. if not os.path.exists(output_dir):
  402. os.makedirs(output_dir)
  403.  
  404. # Determine if input is a GIF or video based on file extension
  405. file_extension = os.path.splitext(input_media)[1].lower()
  406. if file_extension == '.gif':
  407. saved_frame_paths = extract_frames_from_gif(
  408. input_media, output_dir, original_file_name, min_frames_per_slice, max_frames_per_slice,
  409. fps, start_point, end_point, slice_count
  410. )
  411. elif file_extension in ['.mp4', '.mov', '.avi', '.mkv']:
  412. saved_frame_paths = extract_frames_from_video(
  413. input_media, output_dir, original_file_name, min_frames_per_slice, max_frames_per_slice,
  414. fps, start_point, end_point, slice_count
  415. )
  416. else:
  417. raise ValueError("Unsupported media format.")
  418.  
  419. # Load models
  420. load_models()
  421.  
  422. # Adjust frame sizes with aspect ratio preserved
  423. adjusted_frame_paths = adjust_frame_sizes(output_dir, min_size, max_size, global_processor, global_model, global_device)
  424.  
  425. # Unload models
  426. unload_models()
  427.  
  428. print("Media processing complete.")
  429. print(f"Original media: {input_media}")
  430. print(f"Processed frames saved in: {output_dir}")
  431.  
  432. return adjusted_frame_paths # Return the list of adjusted frame paths
  433.  
  434. def simple_process_video(input_video,
  435. output_base_dir,
  436. min_size=(DEFAULT_WIDTH, DEFAULT_HEIGHT),
  437. max_size=(DEFAULT_WIDTH, DEFAULT_HEIGHT),
  438. min_frames_per_slice=DEFAULT_MIN_FRAMES_PER_SLICE,
  439. max_frames_per_slice=DEFAULT_MAX_FRAMES_PER_SLICE,
  440. fps_per_slice=DEFAULT_FPS_PER_SLICE,
  441. similarity_threshold=SIMILARITY_THRESHOLD,
  442. start_point=0.0,
  443. end_point=None,
  444. video_slice_count=1):
  445. """
  446. Simplified function to process a video and return a list of paths to the created images.
  447. """
  448. return process_media(
  449. input_video,
  450. output_base_dir,
  451. min_frames_per_slice,
  452. max_frames_per_slice,
  453. fps_per_slice,
  454. similarity_threshold,
  455. min_size,
  456. max_size,
  457. start_point,
  458. end_point,
  459. video_slice_count
  460. )
  461.  
  462. def simple_process_gif(input_gif,
  463. output_base_dir,
  464. min_size=(DEFAULT_WIDTH, DEFAULT_HEIGHT),
  465. max_size=(DEFAULT_WIDTH, DEFAULT_HEIGHT),
  466. min_frames_per_slice=DEFAULT_MIN_FRAMES_PER_SLICE,
  467. max_frames_per_slice=DEFAULT_MAX_FRAMES_PER_SLICE,
  468. fps_per_slice=DEFAULT_FPS_PER_SLICE,
  469. similarity_threshold=SIMILARITY_THRESHOLD,
  470. start_point=0.0,
  471. end_point=None,
  472. gif_slice_count=1):
  473. """
  474. Simplified function to process a GIF and return a list of paths to the created images.
  475. """
  476. return process_media(
  477. input_gif,
  478. output_base_dir,
  479. min_frames_per_slice,
  480. max_frames_per_slice,
  481. fps_per_slice,
  482. similarity_threshold,
  483. min_size,
  484. max_size,
  485. start_point,
  486. end_point,
  487. gif_slice_count
  488. )
  489.  
  490. # Example usage:
  491. # Uncomment the following lines to test the functions
  492.  
  493. if __name__ == "__main__":
  494. i_start_time = time.time()
  495. i_input_media = 'path_to_your_video_or_gif' # Replace with your media file path
  496. i_output_base_dir = 'output_frames'
  497. i_min_size = (512, 512)
  498. i_max_size = (1024, 1024)
  499. i_min_frames_per_slice = 20
  500. i_max_frames_per_slice = 50
  501. i_fps_per_slice = 10
  502. i_similarity_threshold = 0.6
  503. i_start_point = 0.0
  504. i_end_point = None
  505. i_slice_count = 2
  506.  
  507. # Process media and get the list of image paths
  508. image_paths = process_media(
  509. i_input_media,
  510. i_output_base_dir,
  511. i_min_frames_per_slice,
  512. i_max_frames_per_slice,
  513. i_fps_per_slice,
  514. i_similarity_threshold,
  515. i_min_size,
  516. i_max_size,
  517. i_start_point,
  518. i_end_point,
  519. i_slice_count
  520. )
  521.  
  522. print(f'Processing time: {time.time() - i_start_time:.2f} seconds')
  523. print(f'Generated image paths: {image_paths}')
  524.  
Advertisement
Add Comment
Please, Sign In to add comment