feat: VIDEO_EDIT input type for video trim/crop rich widgets

comfy_api/latest/_input/video_types.py

@@ -4,7 +4,7 @@ from fractions import Fraction
 from typing import Optional, Union, IO
 import io
 import av
-from .._util import VideoContainer, VideoCodec, VideoComponents
+from .._util import VideoContainer, VideoCodec, VideoComponents, normalize_crop_rect
 
 class VideoInput(ABC):
     """
@@ -54,6 +54,45 @@ class VideoInput(ABC):
         """
         pass
 
+    def as_cropped(
+        self,
+        x: int = 0,
+        y: int = 0,
+        width: int = 0,
+        height: int = 0,
+    ) -> VideoInput:
+        """
+        Create a new VideoInput spatially cropped to the given pixel rectangle.
+
+        The rectangle is clamped to the frame and even-aligned for encoder
+        compatibility. An empty or full-frame rectangle returns the input
+        unchanged.
+
+        Default implementation materializes the video via get_components();
+        subclasses should override with lazier strategies when possible.
+        """
+        components = self.get_components()
+        rect = normalize_crop_rect(
+            x, y, width, height, components.images.shape[2], components.images.shape[1]
+        )
+        if rect is None:
+            return self
+        from .._input_impl.video_types import VideoFromComponents
+
+        cx, cy, cw, ch = rect
+        return VideoFromComponents(
+            VideoComponents(
+                images=components.images[:, cy:cy + ch, cx:cx + cw, :],
+                audio=components.audio,
+                frame_rate=components.frame_rate,
+                metadata=components.metadata,
+                alpha=components.alpha[:, cy:cy + ch, cx:cx + cw]
+                if components.alpha is not None
+                else None,
+            ),
+            bit_depth=self.get_bit_depth(),
+        )
+
     def get_stream_source(self) -> Union[str, io.BytesIO]:
         """
         Get a streamable source for the video. This allows processing without

comfy_api/latest/_input_impl/video_types.py

@@ -12,7 +12,7 @@ import numpy as np
 import math
 import os
 import torch
-from .._util import VideoContainer, VideoCodec, VideoComponents
+from .._util import VideoContainer, VideoCodec, VideoComponents, normalize_crop_rect
 import logging
 
 
@@ -115,12 +115,17 @@ def mp4_output_open_kwargs(path: str | io.BytesIO, format: VideoContainer, codec
     return open_kwargs
 
 
+def _rotation_quadrant(frame: av.VideoFrame) -> int:
+    return int(round(frame.rotation // 90)) % 4 if frame.rotation else 0
+
+
 class VideoFromFile(VideoInput):
     """
     Class representing video input from a file.
     """
 
-    def __init__(self, file: str | io.BytesIO, *, start_time: float=0, duration: float=0):
+    def __init__(self, file: str | io.BytesIO, *, start_time: float=0, duration: float=0,
+                 crop: tuple[int, int, int, int] | None = None):
         """
         Initialize the VideoFromFile object based off of either a path on disk or a BytesIO object
         containing the file contents.
@@ -128,6 +133,7 @@ class VideoFromFile(VideoInput):
         self.__file = file
         self.__start_time = start_time
         self.__duration = duration
+        self.__crop = crop
 
     def get_stream_source(self) -> str | io.BytesIO:
         """
@@ -157,7 +163,31 @@ class VideoFromFile(VideoInput):
             for stream in container.streams:
                 if stream.type == 'video':
                     assert isinstance(stream, av.VideoStream)
-                    return stream.width, stream.height
+                    if self.__crop is None:
+                        return stream.width, stream.height
+
+                    display_width, display_height = self._get_display_dimensions()
+                    rect = normalize_crop_rect(*self.__crop, display_width, display_height)
+                    if rect is not None:
+                        return rect[2], rect[3]
+                    return display_width, display_height
+        raise ValueError(f"No video stream found in file '{self.__file}'")
+
+    def _get_display_dimensions(self) -> tuple[int, int]:
+        if isinstance(self.__file, io.BytesIO):
+            self.__file.seek(0)
+        with av.open(self.__file, mode='r') as container:
+            for stream in container.streams:
+                if stream.type == 'video':
+                    assert isinstance(stream, av.VideoStream)
+                    width, height = stream.width, stream.height
+                    try:
+                        frame = next(container.decode(stream), None)
+                    except av.error.FFmpegError:
+                        frame = None
+                    if frame is not None and _rotation_quadrant(frame) % 2:
+                        width, height = height, width
+                    return width, height
         raise ValueError(f"No video stream found in file '{self.__file}'")
 
     def get_bit_depth(self) -> int:
@@ -327,6 +357,8 @@ class VideoFromFile(VideoInput):
         streams = [video_stream]
         has_first_audio_frame = False
         checked_alpha = False
+        crop_rect = None
+        crop_resolved = False
 
         # Default to False so we decode until EOF if duration is 0
         video_done = False
@@ -397,9 +429,16 @@ class VideoFromFile(VideoInput):
                             img = np.ascontiguousarray(align_graph[2].pull().to_ndarray(format=image_format)[:frame.height, :frame.width])
                         else:
                             img = frame.to_ndarray(format=image_format)
-                        if frame.rotation != 0:
-                            k = int(round(frame.rotation // 90))
-                            img = np.rot90(img, k=k, axes=(0, 1)).copy()
+                        rotation_quadrant = _rotation_quadrant(frame)
+                        if rotation_quadrant:
+                            img = np.rot90(img, k=rotation_quadrant, axes=(0, 1)).copy()
+                        if self.__crop is not None:
+                            if not crop_resolved:
+                                crop_rect = normalize_crop_rect(*self.__crop, img.shape[1], img.shape[0])
+                                crop_resolved = True
+                            if crop_rect is not None:
+                                cx, cy, cw, ch = crop_rect
+                                img = np.ascontiguousarray(img[cy:cy + ch, cx:cx + cw])
                         if alphas is None:
                             frames.append(torch.from_numpy(img))
                         else:
@@ -484,6 +523,8 @@ class VideoFromFile(VideoInput):
                 reuse_streams = False
             if self.__start_time or self.__duration:
                 reuse_streams = False
+            if self.__crop is not None:
+                reuse_streams = False
 
             if not reuse_streams:
                 if bit_depth is None:
@@ -564,6 +605,12 @@ class VideoFromFile(VideoInput):
             if duration:
                 duration_cap = math.ceil(duration * sample_rate)
 
+        import comfy.utils
+        raw_duration = self._get_raw_duration()
+        window_seconds = duration if duration else max(raw_duration - start_time, 0.0)
+        progress_total = max(1, int(round(window_seconds * float(rate))))
+        pbar = comfy.utils.ProgressBar(progress_total)
+
         streams = [video_stream] if audio_stream is None else [video_stream, audio_stream]
         pts_step = max(1, int(round((1 / rate) / video_stream.time_base)))
         video_done = False
@@ -576,6 +623,8 @@ class VideoFromFile(VideoInput):
         source_size = None
         rotation_k = 0
         rotation_filter = None
+        crop_rect = None
+        crop_filter = None
         audio_started = False
         samples_written = 0
         pending_audio = []
@@ -649,11 +698,15 @@ class VideoFromFile(VideoInput):
                         if end_pts is not None and frame.pts is not None:
                             frame_duration = min(frame_duration, end_pts - frame.pts)
                         if output is None:
-                            rotation_k = int(round(frame.rotation // 90)) % 4 if frame.rotation else 0
+                            rotation_k = _rotation_quadrant(frame)
                             if rotation_k % 2:
                                 out_width, out_height = frame.height, frame.width
                             else:
                                 out_width, out_height = frame.width, frame.height
+                            if self.__crop is not None:
+                                crop_rect = normalize_crop_rect(*self.__crop, out_width, out_height)
+                                if crop_rect is not None:
+                                    out_width, out_height = crop_rect[2], crop_rect[3]
                             if out_width % 2 or out_height % 2:
                                 raise ValueError(f"H.264 output requires even dimensions, got {out_width}x{out_height}")
                             source_size = (frame.width, frame.height)
@@ -694,9 +747,22 @@ class VideoFromFile(VideoInput):
                                 g_sink = g.add("buffersink")
                                 tail.link_to(g_sink)
                                 g.configure()
-                                rotation_filter = (g_src, g_sink)
-                            rotation_filter[0].push(frame)
-                            frame = rotation_filter[1].pull()
+                                rotation_filter = (g, g_src, g_sink)
+                            rotation_filter[1].push(frame)
+                            frame = rotation_filter[2].pull()
+                        if crop_rect is not None:
+                            if crop_filter is None:
+                                g = av.filter.Graph()
+                                g_src = g.add_buffer(width=frame.width, height=frame.height,
+                                                     format=frame.format.name, time_base=video_stream.time_base)
+                                g_crop = g.add("crop", f"{crop_rect[2]}:{crop_rect[3]}:{crop_rect[0]}:{crop_rect[1]}")
+                                g_sink = g.add("buffersink")
+                                g_src.link_to(g_crop)
+                                g_crop.link_to(g_sink)
+                                g.configure()
+                                crop_filter = (g, g_src, g_sink)
+                            crop_filter[1].push(frame)
+                            frame = crop_filter[2].pull()
                         if frame.color_range == ColorRange.JPEG:
                             # compress full-range sources (yuvj/MJPEG) to limited range
                             frame = frame.reformat(format=pix_fmt, src_color_range="JPEG", dst_color_range="MPEG")
@@ -735,6 +801,7 @@ class VideoFromFile(VideoInput):
                             out_packet.duration = video_frame_durations.pop(out_packet.pts, 0)
                             output.mux(out_packet)
                         drain_audio()
+                        pbar.update(1)
 
                 elif packet.stream == audio_stream and not audio_done:
                     for resampled in itertools.chain.from_iterable(map(resampler.resample, packet.decode())):
@@ -804,11 +871,42 @@ class VideoFromFile(VideoInput):
             self.get_stream_source(),
             start_time=start_time + self.__start_time,
             duration=duration,
+            crop=self.__crop,
         )
         if trimmed.get_duration() < duration and strict_duration:
             return None
         return trimmed
 
+    def as_cropped(
+        self, x: int = 0, y: int = 0, width: int = 0, height: int = 0
+    ) -> VideoInput:
+        if int(width) <= 0 or int(height) <= 0:
+            return self
+
+        display_width, display_height = self._get_display_dimensions()
+        outer = (
+            normalize_crop_rect(*self.__crop, display_width, display_height)
+            if self.__crop is not None
+            else None
+        )
+        if outer is None:
+            rect = normalize_crop_rect(x, y, width, height, display_width, display_height)
+        else:
+            inner = normalize_crop_rect(x, y, width, height, outer[2], outer[3])
+            rect = (
+                (outer[0] + inner[0], outer[1] + inner[1], inner[2], inner[3])
+                if inner is not None
+                else None
+            )
+        if rect is None:
+            return self
+        return VideoFromFile(
+            self.get_stream_source(),
+            start_time=self.__start_time,
+            duration=self.__duration,
+            crop=rect,
+        )
+
 
 class VideoFromComponents(VideoInput):
     """
@@ -825,6 +923,8 @@ class VideoFromComponents(VideoInput):
             images=self.__components.images,
             audio=self.__components.audio,
             frame_rate=self.__components.frame_rate,
+            metadata=self.__components.metadata,
+            alpha=self.__components.alpha,
         )
 
     def get_bit_depth(self) -> int:

comfy_api/latest/_io.py

@@ -1416,6 +1416,41 @@ class BoundingBoxes(ComfyTypeIO):
                 self.default = []
 
 
+@comfytype(io_type="VIDEO_EDIT")
+class VideoEdit(ComfyTypeIO):
+    class VideoTrimSection(TypedDict):
+        start_time: float
+        duration: float
+
+    class VideoCropSection(TypedDict):
+        x: int
+        y: int
+        width: int
+        height: int
+
+    class VideoEditDict(TypedDict, total=False):
+        trim: 'VideoEdit.VideoTrimSection'
+        crop: 'VideoEdit.VideoCropSection'
+    Type = VideoEditDict
+
+    class Input(WidgetInput):
+        def __init__(self, id: str, display_name: str=None, optional=False, tooltip: str=None,
+                     socketless: bool=True, default: dict=None, features: list[str]=None, advanced: bool=None):
+            super().__init__(id, display_name, optional, tooltip, None, default, socketless, None, None, None, None, advanced)
+            self.features = features if features is not None else ["trim", "crop"]
+            if default is None:
+                self.default = {}
+                if "trim" in self.features:
+                    self.default["trim"] = {"start_time": 0.0, "duration": 0.0}
+                if "crop" in self.features:
+                    self.default["crop"] = {"x": 0, "y": 0, "width": 0, "height": 0}
+
+        def as_dict(self):
+            return super().as_dict() | prune_dict({
+                "features": self.features,
+            })
+
+
 @comfytype(io_type="HISTOGRAM")
 class Histogram(ComfyTypeIO):
     """A histogram represented as a list of bin counts."""
@@ -2493,5 +2528,6 @@ __all__ = [
     "Curve",
     "Histogram",
     "Range",
+    "VideoEdit",
     "NodeReplace",
 ]

comfy_api/latest/_util/__init__.py

@@ -1,4 +1,4 @@
-from .video_types import VideoContainer, VideoCodec, VideoComponents
+from .video_types import VideoContainer, VideoCodec, VideoComponents, normalize_crop_rect
 from .geometry_types import VOXEL, MESH, SPLAT, File3D
 from .image_types import SVG
 
@@ -7,6 +7,7 @@ __all__ = [
     "VideoContainer",
     "VideoCodec",
     "VideoComponents",
+    "normalize_crop_rect",
     "VOXEL",
     "MESH",
     "SPLAT",

comfy_api/latest/_util/video_types.py

@@ -48,3 +48,23 @@ class VideoComponents:
     audio: Optional[AudioInput] = None
     metadata: Optional[dict] = None
     alpha: Optional[MaskInput] = None
+
+
+def normalize_crop_rect(
+    x: int, y: int, width: int, height: int, source_width: int, source_height: int
+) -> Optional[tuple[int, int, int, int]]:
+    width = int(width)
+    height = int(height)
+    if width <= 0 or height <= 0:
+        return None
+    x = max(0, min(int(x), source_width - 1))
+    y = max(0, min(int(y), source_height - 1))
+    width = min(width, source_width - x)
+    height = min(height, source_height - y)
+    if x == 0 and y == 0 and width == source_width and height == source_height:
+        return None
+    width -= width % 2
+    height -= height % 2
+    if width <= 0 or height <= 0:
+        return None
+    return x, y, width, height