Whisper word timestamps from 4 engines, cut to one clip and paged 5 words at a time

curl -O https://api.clipsdk.com/f/lib/demo/artemis-moon-speech.mp4
ffmpeg -i artemis-moon-speech.mp4 -t 120 -vn -ac 1 -ar 16000 talk.wav
pip install "faster-whisper==1.2.1" "av==15.1.0"  # av 19.0.2 breaks faster-whisper 1.2.1

import json
from faster_whisper import WhisperModel

model = WhisperModel("large-v3-turbo", device="cpu", compute_type="int8")
segments, info = model.transcribe("talk.wav", word_timestamps=True, language="en")

words = [{"word": w.word.strip(), "start": w.start, "end": w.end} for s in segments for w in s.words]
json.dump(words, open("words.json", "w"))

for w in words[:6]:
    print(f'{w["start"]:6.2f} {w["end"]:6.2f} {w["word"]}')
print(len(words), "words")

# $ python faster_whisper_words.py
#   0.00   0.34 Throughout
#   0.34   1.28 America's
#   1.28   1.86 story,
#   2.50   2.62 there
#   2.62   2.92 are
#   2.92   3.62 defining
# 209 words

import json

CLIP_START, CLIP_END = 49.51, 71.24
MAX_WORDS, PAUSE_S = 5, 0.45

words = json.load(open("words.json"))

# A word belongs to the clip its midpoint falls in, and its times become clip-relative
clip = [
    {"word": w["word"], "start": max(w["start"], CLIP_START) - CLIP_START, "end": min(w["end"], CLIP_END) - CLIP_START}
    for w in words
    if CLIP_START <= (w["start"] + w["end"]) / 2 < CLIP_END
]

# New caption page on a full page, a pause or a sentence end
pages, page = [], []
for w in clip:
    if page and (len(page) == MAX_WORDS or w["start"] - page[-1]["end"] > PAUSE_S or page[-1]["word"][-1] in ".?!"):
        pages.append(page)
        page = []
    page.append(w)
pages.append(page)


def stamp(t):
    ms = round(t * 1000)
    return f"{ms // 3600000:02}:{ms // 60000 % 60:02}:{ms // 1000 % 60:02},{ms % 1000:03}"


def karaoke(page):
    parts = []
    for i, w in enumerate(page):
        until = page[i + 1]["start"] if i + 1 < len(page) else w["end"]
        parts.append(f'{{\\k{round((until - w["start"]) * 100)}}}{w["word"]}')
    return " ".join(parts)


with open("clip.srt", "w") as srt:
    for i, p in enumerate(pages, 1):
        srt.write(f'{i}\n{stamp(p[0]["start"])} --> {stamp(p[-1]["end"])}\n{" ".join(w["word"] for w in p)}\n\n')

print(len(clip), "words,", len(pages), "pages")
print("\n\n".join(open("clip.srt").read().split("\n\n")[:3]))
print(karaoke(pages[0]))

# $ python clip_captions.py
# 43 words, 9 pages
# 1
# 00:00:00,210 --> 00:00:02,930
# President Kennedy knew that vision
#
# 2
# 00:00:02,930 --> 00:00:05,010
# would be hard, not easy.
#
# 3
# 00:00:05,310 --> 00:00:06,610
# He made that clear.
# {\k82}President {\k54}Kennedy {\k44}knew {\k44}that {\k48}vision