Face tracking crop: 1920x1080 in, 1080x1920 out, with YuNet and ffmpeg

curl -O https://api.clipsdk.com/f/lib/demo/thermal-energy-lesson.mp4
curl -LO https://github.com/opencv/opencv_zoo/raw/main/models/face_detection_yunet/face_detection_yunet_2023mar.onnx
pip install "opencv-python-headless==5.0.0.93"

import json
import cv2

CLIP_END = 55.78

detector = cv2.FaceDetectorYN.create("face_detection_yunet_2023mar.onnx", "", (640, 360), 0.6, 0.3, 16)
video = cv2.VideoCapture("thermal-energy-lesson.mp4")
fps = video.get(cv2.CAP_PROP_FPS)
step = round(fps / 5)

samples = []
frame_no = 0
while frame_no / fps < CLIP_END:
    ok, frame = video.read()
    if not ok:
        break
    if frame_no % step == 0:
        _, faces = detector.detect(cv2.resize(frame, (640, 360)))
        cx = None
        if faces is not None:
            x, y, w, h = max(faces, key=lambda f: f[2] * f[3])[:4]
            cx = float(x + w / 2) * frame.shape[1] / 640
        samples.append({"t": round(frame_no / fps, 2), "cx": cx})
    frame_no += 1

json.dump(samples, open("faces.json", "w"))
print(len(samples), "samples,", sum(s["cx"] is not None for s in samples), "with a face")

# $ python faces.py
# 279 samples, 171 with a face

import json

SOURCE_W, CROP_W = 1920, 608
DEAD_ZONE = 0.12 * CROP_W
MAX_MOVE = 0.5 * CROP_W * 0.2

samples = json.load(open("faces.json"))
centre_x = (SOURCE_W - CROP_W) / 2

x, lines, missed = None, [], 0
for s in samples:
    if s["cx"] is None:
        continue
    if not centre_x < s["cx"] < centre_x + CROP_W:
        missed += 1
    target = min(max(s["cx"] - CROP_W / 2, 0), SOURCE_W - CROP_W)

    # Snap on the first face and on a jump of half a crop (a cut), else ease past the dead zone
    if x is None or abs(target - x) > CROP_W / 2:
        x = target
    elif abs(target - x) > DEAD_ZONE:
        move = min(abs(target - x) - DEAD_ZONE, MAX_MOVE)
        x += move if target > x else -move
    lines.append(f'{s["t"]:.2f} crop x {round(x)};')

open("crop.cmd", "w").write("\n".join(lines) + "\n")
print(len(lines), "commands, a centre crop misses the face in", missed)
print("\n".join(lines[:4]))

# $ python crop_path.py
# 171 commands, a centre crop misses the face in 149
# 0.20 crop x 306;
# 0.40 crop x 306;
# 0.60 crop x 306;
# 0.80 crop x 306;

ffmpeg -t 55.78 -i thermal-energy-lesson.mp4 -vf "sendcmd=f=crop.cmd,crop=608:1080:656:0,scale=1080:1920" -c:v libx264 -preset veryfast -c:a aac tracked.mp4
ffprobe -v error -select_streams v -show_entries stream=width,height:format=duration -of compact=p=0 tracked.mp4

# output:
# width=1080|height=1920
# duration=55.800000