AutoFlip alternative: rebuild the 9:16 crop in Python

python3 -m venv venv && . venv/bin/activate
pip install -q mediapipe==1.2.2 scenedetect==0.7.2
curl -sO https://api.clipsdk.com/f/lib/demo/rocket-ranch-podcast.mp4
for model in full_range short_range; do
  curl -sO https://storage.googleapis.com/mediapipe-models/face_detector/blaze_face_$model/float16/1/blaze_face_$model.tflite
done
ffmpeg -v error -ss 186.31 -i rocket-ranch-podcast.mp4 -t 59.13 -c:v libx264 -crf 18 -c:a aac moment.mp4


# $ pip show -f mediapipe | grep -i -c autoflip
# 0


# $ python3 --version && ffmpeg -version | head -1
# Python 3.13.14
# ffmpeg version 7.1.5-0+deb13u1 Copyright (c) 2000-2026 the FFmpeg developers

import subprocess
import cv2
import mediapipe as mp
from mediapipe.tasks.python import BaseOptions, vision
from scenedetect import ContentDetector, detect

SRC, OUT = "moment.mp4", "vertical.mp4"

# AutoFlip step 1: split the video into shots at every camera cut
shots = detect(SRC, ContentDetector())

video = cv2.VideoCapture(SRC)
W = int(video.get(cv2.CAP_PROP_FRAME_WIDTH))
H = int(video.get(cv2.CAP_PROP_FRAME_HEIGHT))
crop_w = round(H * 9 / 16 / 2) * 2
faces = vision.FaceDetector.create_from_options(vision.FaceDetectorOptions(
    base_options=BaseOptions(model_asset_path="blaze_face_full_range.tflite")))

def biggest_face_x(t):
    video.set(cv2.CAP_PROP_POS_MSEC, t * 1000)
    ok, frame = video.read()
    image = mp.Image(image_format=mp.ImageFormat.SRGB, data=cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
    found = faces.detect(image).detections
    if not found:
        return None
    box = max(found, key=lambda d: d.bounding_box.width).bounding_box
    return box.origin_x + box.width / 2

# AutoFlip step 2: one steady crop per shot, its "stable camera" choice
plan = []
for start, end in shots:
    a, b = start.seconds, end.seconds
    xs = [x for i in range(5) if (x := biggest_face_x(a + (b - a) * (i + 0.5) / 5)) is not None]
    centre = sorted(xs)[len(xs) // 2] if xs else W / 2
    left = int(min(max(centre - crop_w / 2, 0), W - crop_w))
    plan.append((b, left))
    print(f"shot {a:5.2f}-{b:5.2f}s  face in {len(xs)}/5 samples  crop x={left}")

# AutoFlip step 3: render, switching the crop at each cut
x_expr = str(plan[-1][1])
for end, left in reversed(plan[:-1]):
    x_expr = f"if(lt(t,{end:.3f}),{left},{x_expr})"
subprocess.run(["ffmpeg", "-v", "error", "-y", "-i", SRC,
                "-vf", f"crop={crop_w}:{H}:'{x_expr}':0,scale=1080:1920",
                "-c:a", "copy", OUT], check=True)
print(f"{len(plan)} shots, {crop_w}x{H} crop scaled to 1080x1920: {OUT}")

# $ time python3 reframe.py 2>/dev/null
# shot  0.00- 5.47s  face in 5/5 samples  crop x=1256
# shot  5.47-17.73s  face in 5/5 samples  crop x=476
# shot 17.73-23.13s  face in 5/5 samples  crop x=818
# shot 23.13-30.73s  face in 5/5 samples  crop x=41
# shot 30.73-35.73s  face in 5/5 samples  crop x=474
# shot 35.73-39.23s  face in 5/5 samples  crop x=1257
# shot 39.23-50.73s  face in 5/5 samples  crop x=426
# shot 50.73-59.10s  face in 4/5 samples  crop x=38
# 8 shots, 608x1080 crop scaled to 1080x1920: vertical.mp4
#
# real	2m32.576s
# user	3m57.115s
# sys	0m12.353s

import cv2
import mediapipe as mp
from mediapipe.tasks.python import BaseOptions, vision

video = cv2.VideoCapture("moment.mp4")
detectors = {
    name: vision.FaceDetector.create_from_options(vision.FaceDetectorOptions(
        base_options=BaseOptions(model_asset_path=f"blaze_face_{name}.tflite")))
    for name in ("short_range", "full_range")
}

hits = dict.fromkeys(detectors, 0)
for second in range(59):
    video.set(cv2.CAP_PROP_POS_MSEC, second * 1000 + 500)
    ok, frame = video.read()
    image = mp.Image(image_format=mp.ImageFormat.SRGB, data=cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
    for name, detector in detectors.items():
        hits[name] += len(detector.detect(image).detections) > 0

print({name: f"face found in {count}/59 frames" for name, count in hits.items()})

# $ python3 compare_detectors.py 2>/dev/null
# {'short_range': 'face found in 35/59 frames', 'full_range': 'face found in 59/59 frames'}