Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import time
- import torch
- from PIL import Image, PngImagePlugin
- import os
- import numpy as np
- from skimage.metrics import structural_similarity as ssim
- import cv2
- import math
- from typing import List
- # Constants
- DEFAULT_MIN_FRAMES_PER_SLICE = 10 # Minimum frames to extract per slice
- DEFAULT_MAX_FRAMES_PER_SLICE = 100 # Maximum frames to extract per slice
- DEFAULT_FPS_PER_SLICE = 10 # Frames per second to extract per slice
- SIMILARITY_THRESHOLD = 0.5
- DEFAULT_WIDTH = 1024
- DEFAULT_HEIGHT = 1024
- UPSCALE_MODEL_NAME = "caidas/swin2SR-lightweight-x2-64"
- # Global variables for models and device
- global_processor = None
- global_model = None
- global_device = None
- def load_models():
- """
- Loads the Swin2SR model and processor.
- """
- global global_processor
- global global_model
- global global_device
- from transformers import Swin2SRImageProcessor, Swin2SRForImageSuperResolution
- print("Loading models...")
- model_name = UPSCALE_MODEL_NAME
- global_processor = Swin2SRImageProcessor.from_pretrained(model_name)
- global_model = Swin2SRForImageSuperResolution.from_pretrained(model_name)
- global_model.eval()
- global_device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
- global_model.to(global_device)
- print("Models loaded.")
- def unload_models():
- """
- Unloads the models and clears GPU memory.
- """
- global global_processor
- global global_model
- global global_device
- print("Unloading models...")
- if global_model is not None:
- del global_model
- if global_processor is not None:
- del global_processor
- torch.cuda.empty_cache()
- print("Models unloaded.")
- def extract_frames_from_video(input_video, output_dir, original_file_name, min_frames_per_slice, max_frames_per_slice,
- fps=DEFAULT_FPS_PER_SLICE, start_point=0.0, end_point=None, video_slice_count=1):
- """
- Extract frames from video with control over frame extraction rate, slicing, and a limit on total images.
- Returns a list of file paths to the saved frames.
- """
- cap = cv2.VideoCapture(input_video)
- video_fps = cap.get(cv2.CAP_PROP_FPS)
- if video_fps == 0:
- video_fps = fps # Default to requested fps if video fps is unavailable
- video_frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
- video_duration = video_frame_count / video_fps # in seconds
- # Set default end_point if not provided
- if end_point is None or end_point > video_duration:
- end_point = video_duration
- # Use floor and ceil for start and end points
- start_point = max(0.0, math.floor(start_point))
- end_point = min(video_duration, math.ceil(end_point))
- # Calculate slice durations
- total_duration = end_point - start_point
- if video_slice_count != 0:
- slice_duration = total_duration / video_slice_count
- else:
- slice_duration = total_duration
- saved_frame_paths = []
- for slice_idx in range(video_slice_count):
- slice_start = start_point + slice_idx * slice_duration
- slice_end = min(start_point + (slice_idx + 1) * slice_duration, end_point)
- # Set the video to the start of the slice
- cap.set(cv2.CAP_PROP_POS_MSEC, slice_start * 1000)
- frames = []
- extracted_frames = 0
- frame_count = 0
- while cap.isOpened():
- current_time = cap.get(cv2.CAP_PROP_POS_MSEC) / 1000 # Current time in seconds
- if current_time > slice_end:
- break
- ret, frame = cap.read()
- if not ret:
- break
- if frame_count % max(int(video_fps / fps), 1) == 0:
- # Convert frame to RGB
- frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
- frames.append((frame_rgb, current_time))
- extracted_frames += 1
- frame_count += 1
- if frames:
- # Adjust frames to meet min and max frame requirements
- frames = adjust_frame_count(frames, min_frames_per_slice, max_frames_per_slice, SIMILARITY_THRESHOLD)
- # Save frames with metadata
- frame_paths = save_frames(frames, output_dir, original_file_name, slice_idx, start_point=slice_start)
- saved_frame_paths.extend(frame_paths)
- cap.release()
- print(f"Extracted and saved frames from video.")
- return saved_frame_paths
- def extract_frames_from_gif(input_gif, output_dir, original_file_name, min_frames_per_slice, max_frames_per_slice,
- fps=DEFAULT_FPS_PER_SLICE, start_point=0.0, end_point=None, gif_slice_count=1):
- """
- Extract frames from GIF with control over frame extraction rate, slicing, and a limit on total images.
- Returns a list of file paths to the saved frames.
- """
- gif = Image.open(input_gif)
- gif_duration = gif.info.get('duration', 100) # Duration of each frame in milliseconds
- total_frames = gif.n_frames
- total_duration = (gif_duration * total_frames) / 1000.0 # in seconds
- # Set default end_point if not provided
- if end_point is None or end_point > total_duration:
- end_point = total_duration
- # Use floor and ceil for start and end points
- start_point = max(0.0, math.floor(start_point))
- end_point = min(total_duration, math.ceil(end_point))
- # Calculate slice durations
- total_duration = end_point - start_point
- slice_duration = total_duration / gif_slice_count
- saved_frame_paths = []
- for slice_idx in range(gif_slice_count):
- slice_start = start_point + slice_idx * slice_duration
- slice_end = min(start_point + (slice_idx + 1) * slice_duration, end_point)
- frames = []
- extracted_frames = 0
- frame_index = 0
- current_time = 0.0
- try:
- gif.seek(0)
- while True:
- current_time = (gif.tell() * gif_duration) / 1000.0 # Current time in seconds
- if current_time >= slice_start and current_time <= slice_end:
- frame = gif.convert('RGB')
- frames.append((np.array(frame), current_time))
- extracted_frames += 1
- else:
- if current_time > slice_end:
- break
- gif.seek(gif.tell() + 1)
- frame_index += 1
- except EOFError:
- pass
- if frames:
- # Adjust frames to meet min and max frame requirements
- frames = adjust_frame_count(frames, min_frames_per_slice, max_frames_per_slice, SIMILARITY_THRESHOLD)
- # Save frames with metadata
- frame_paths = save_frames(frames, output_dir, original_file_name, slice_idx, start_point=slice_start)
- saved_frame_paths.extend(frame_paths)
- print(f"Extracted and saved frames from GIF.")
- return saved_frame_paths
- def adjust_frame_count(frames_with_times, min_frames, max_frames, similarity_threshold):
- """
- Adjusts the number of frames to be within min_frames and max_frames by interpolating or reducing frames.
- """
- current_total_frames = len(frames_with_times)
- if current_total_frames < min_frames:
- frames_with_times = interpolate_frames(frames_with_times, min_frames - current_total_frames, similarity_threshold)
- elif current_total_frames > max_frames:
- frames_with_times = reduce_frames(frames_with_times, max_frames)
- return frames_with_times
- def interpolate_frames(frames_with_times, frames_needed, similarity_threshold):
- """
- Interpolates frames to increase the total number of frames.
- """
- print(f"Interpolating {frames_needed} frames to meet minimum frame requirement.")
- interpolation_candidates = []
- for i in range(len(frames_with_times) - 1):
- current_frame, _ = frames_with_times[i]
- next_frame, _ = frames_with_times[i + 1]
- # Convert to grayscale for SSIM comparison
- gray_current = cv2.cvtColor(current_frame, cv2.COLOR_RGB2GRAY)
- gray_next = cv2.cvtColor(next_frame, cv2.COLOR_RGB2GRAY)
- # Calculate SSIM (Structural Similarity Index)
- score = ssim(gray_current, gray_next)
- # Interpolate if the frames are similar enough
- if score >= similarity_threshold:
- interpolation_candidates.append(i)
- if not interpolation_candidates:
- print("No suitable frames for interpolation.")
- return frames_with_times
- num_interpolations = min(len(interpolation_candidates), frames_needed)
- indices_to_interpolate = np.linspace(0, len(interpolation_candidates) - 1, num_interpolations, dtype=int)
- indices_to_interpolate = [interpolation_candidates[i] for i in indices_to_interpolate]
- interpolated = []
- for i in range(len(frames_with_times)):
- interpolated.append(frames_with_times[i])
- if i in indices_to_interpolate and i + 1 < len(frames_with_times):
- current_frame, current_time = frames_with_times[i]
- next_frame, next_time = frames_with_times[i + 1]
- blended_frame = np.uint8((current_frame.astype(np.float32) + next_frame.astype(np.float32)) / 2)
- blended_time = (current_time + next_time) / 2
- interpolated.append((blended_frame, blended_time))
- return interpolated
- def reduce_frames(frames_with_times, max_frames):
- """
- Reduces the number of frames to meet the maximum frame limit.
- """
- print(f"Reducing frames to meet maximum frame limit of {max_frames}.")
- total_frames = len(frames_with_times)
- indices_to_keep = np.linspace(0, total_frames - 1, max_frames, dtype=int)
- reduced_frames = [frames_with_times[i] for i in indices_to_keep]
- return reduced_frames
- def save_frames(frames_with_times, output_dir, original_file_name, slice_number, start_point=0.0):
- """
- Saves frames to the output directory with sequential numerical filenames and metadata.
- Returns a list of file paths to the saved frames.
- """
- saved_frame_paths = []
- if not os.path.exists(output_dir):
- os.makedirs(output_dir)
- for idx, (frame, frame_time) in enumerate(frames_with_times):
- img = Image.fromarray(frame)
- frame_number = idx
- frame_filename = f"{original_file_name}_slice_{slice_number}_frame_{frame_number}.png"
- frame_path = os.path.join(output_dir, frame_filename)
- # Prepare metadata
- metadata = PngImagePlugin.PngInfo()
- metadata.add_text("Original Media", original_file_name)
- metadata.add_text("Slice Number", str(slice_number))
- metadata.add_text("Frame Count", str(len(frames_with_times)))
- metadata.add_text("Frame Time (ms)", str(int(frame_time * 1000)))
- if idx % 2 == 1:
- # This frame is interpolated
- metadata.add_text("Interpolated", "True")
- previous_time = frames_with_times[idx - 1][1]
- time_difference = frame_time - previous_time
- metadata.add_text("Time Difference (ms)", str(int(time_difference * 1000)))
- else:
- metadata.add_text("Interpolated", "False")
- img.save(frame_path, pnginfo=metadata)
- saved_frame_paths.append(frame_path)
- return saved_frame_paths
- def resize_with_aspect_ratio(img, min_size, max_size):
- """
- Resizes an image to fit within min_size and max_size while maintaining aspect ratio.
- """
- width, height = img.size
- aspect_ratio = width / height
- # Calculate scaling factors
- width_scale_min = min_size[0] / width
- height_scale_min = min_size[1] / height
- width_scale_max = max_size[0] / width
- height_scale_max = max_size[1] / height
- # Determine the scaling factor for upscaling or downscaling
- scale_factor = 1.0
- # Upscale if necessary
- if width < min_size[0] or height < min_size[1]:
- scale_factor = max(width_scale_min, height_scale_min)
- # Downscale if necessary
- if width > max_size[0] or height > max_size[1]:
- scale_factor = min(width_scale_max, height_scale_max)
- # Apply scaling if needed
- if scale_factor != 1.0:
- new_width = int(width * scale_factor)
- new_height = int(height * scale_factor)
- img = img.resize((new_width, new_height), resample=Image.Resampling.LANCZOS)
- return img
- def adjust_frame_sizes(output_dir, min_size, max_size, processor, model, device):
- """
- Adjusts the size of all frames in the output directory to fit within min_size and max_size while maintaining aspect ratio.
- Returns a list of file paths to the resized images.
- """
- print("Adjusting frame sizes...")
- resized_frame_paths = []
- for filename in os.listdir(output_dir):
- if filename.endswith('.png'):
- img_path = os.path.join(output_dir, filename)
- img = Image.open(img_path).convert('RGB')
- width, height = img.size
- # Determine if upscaling or downscaling is needed
- need_upscale = width < min_size[0] or height < min_size[1]
- need_downscale = width > max_size[0] or height > max_size[1]
- # Upscale if necessary
- if need_upscale:
- img = upscale_image(img, model, processor, device)
- print(f"Upscaled {filename} to meet minimum size {min_size}.")
- # After upscaling, update width and height
- width, height = img.size
- # Resize to fit within min_size and max_size while preserving aspect ratio
- img = resize_with_aspect_ratio(img, min_size, max_size)
- # Save the processed image back
- img.save(img_path)
- resized_frame_paths.append(img_path)
- print(f"Size adjustment complete. Frames saved in {output_dir}.")
- return resized_frame_paths
- def upscale_image(img, model, processor, device):
- """
- Upscales an image using the specified Hugging Face model.
- """
- # Preprocess the image
- inputs = processor(images=img, return_tensors="pt").to(device)
- # Perform inference
- with torch.no_grad():
- outputs = model(**inputs)
- # Post-process the output
- output = outputs.reconstruction.data.squeeze().clamp(0, 1).cpu().numpy()
- output = (output * 255.0).astype(np.uint8)
- output = np.transpose(output, (1, 2, 0))
- upscaled_img = Image.fromarray(output)
- return upscaled_img
- def process_media(
- input_media,
- output_base_dir,
- min_frames_per_slice,
- max_frames_per_slice,
- fps,
- similarity_threshold,
- min_size,
- max_size,
- start_point=0.0,
- end_point=None,
- slice_count=1
- ):
- """
- Processes a media file (video or GIF) and returns a list of paths to the created images.
- """
- original_file_name = os.path.splitext(os.path.basename(input_media))[0]
- # Create a unique output directory per input media file
- timestamp = time.strftime("%Y%m%d-%H%M%S")
- output_dir = os.path.join(output_base_dir, f"{original_file_name}_{timestamp}")
- if not os.path.exists(output_dir):
- os.makedirs(output_dir)
- # Determine if input is a GIF or video based on file extension
- file_extension = os.path.splitext(input_media)[1].lower()
- if file_extension == '.gif':
- saved_frame_paths = extract_frames_from_gif(
- input_media, output_dir, original_file_name, min_frames_per_slice, max_frames_per_slice,
- fps, start_point, end_point, slice_count
- )
- elif file_extension in ['.mp4', '.mov', '.avi', '.mkv']:
- saved_frame_paths = extract_frames_from_video(
- input_media, output_dir, original_file_name, min_frames_per_slice, max_frames_per_slice,
- fps, start_point, end_point, slice_count
- )
- else:
- raise ValueError("Unsupported media format.")
- # Load models
- load_models()
- # Adjust frame sizes with aspect ratio preserved
- adjusted_frame_paths = adjust_frame_sizes(output_dir, min_size, max_size, global_processor, global_model, global_device)
- # Unload models
- unload_models()
- print("Media processing complete.")
- print(f"Original media: {input_media}")
- print(f"Processed frames saved in: {output_dir}")
- return adjusted_frame_paths # Return the list of adjusted frame paths
- def simple_process_video(input_video,
- output_base_dir,
- min_size=(DEFAULT_WIDTH, DEFAULT_HEIGHT),
- max_size=(DEFAULT_WIDTH, DEFAULT_HEIGHT),
- min_frames_per_slice=DEFAULT_MIN_FRAMES_PER_SLICE,
- max_frames_per_slice=DEFAULT_MAX_FRAMES_PER_SLICE,
- fps_per_slice=DEFAULT_FPS_PER_SLICE,
- similarity_threshold=SIMILARITY_THRESHOLD,
- start_point=0.0,
- end_point=None,
- video_slice_count=1):
- """
- Simplified function to process a video and return a list of paths to the created images.
- """
- return process_media(
- input_video,
- output_base_dir,
- min_frames_per_slice,
- max_frames_per_slice,
- fps_per_slice,
- similarity_threshold,
- min_size,
- max_size,
- start_point,
- end_point,
- video_slice_count
- )
- def simple_process_gif(input_gif,
- output_base_dir,
- min_size=(DEFAULT_WIDTH, DEFAULT_HEIGHT),
- max_size=(DEFAULT_WIDTH, DEFAULT_HEIGHT),
- min_frames_per_slice=DEFAULT_MIN_FRAMES_PER_SLICE,
- max_frames_per_slice=DEFAULT_MAX_FRAMES_PER_SLICE,
- fps_per_slice=DEFAULT_FPS_PER_SLICE,
- similarity_threshold=SIMILARITY_THRESHOLD,
- start_point=0.0,
- end_point=None,
- gif_slice_count=1):
- """
- Simplified function to process a GIF and return a list of paths to the created images.
- """
- return process_media(
- input_gif,
- output_base_dir,
- min_frames_per_slice,
- max_frames_per_slice,
- fps_per_slice,
- similarity_threshold,
- min_size,
- max_size,
- start_point,
- end_point,
- gif_slice_count
- )
- # Example usage:
- # Uncomment the following lines to test the functions
- if __name__ == "__main__":
- i_start_time = time.time()
- i_input_media = 'path_to_your_video_or_gif' # Replace with your media file path
- i_output_base_dir = 'output_frames'
- i_min_size = (512, 512)
- i_max_size = (1024, 1024)
- i_min_frames_per_slice = 20
- i_max_frames_per_slice = 50
- i_fps_per_slice = 10
- i_similarity_threshold = 0.6
- i_start_point = 0.0
- i_end_point = None
- i_slice_count = 2
- # Process media and get the list of image paths
- image_paths = process_media(
- i_input_media,
- i_output_base_dir,
- i_min_frames_per_slice,
- i_max_frames_per_slice,
- i_fps_per_slice,
- i_similarity_threshold,
- i_min_size,
- i_max_size,
- i_start_point,
- i_end_point,
- i_slice_count
- )
- print(f'Processing time: {time.time() - i_start_time:.2f} seconds')
- print(f'Generated image paths: {image_paths}')
Advertisement
Add Comment
Please, Sign In to add comment