diff --git a/bin/generate_data.sh b/bin/generate_data.sh index 5dbe4d4..74ed5f8 100755 --- a/bin/generate_data.sh +++ b/bin/generate_data.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash set -euo pipefail +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +STATE_FILE="${SCRIPT_DIR}/.personality_index" + IN_DIR="${1:?Usage: $0 [\"Title\"] [\"Location\"] [max_frames]}" TITLE="${2:-}" LOCATION="${3:-}" @@ -8,27 +11,30 @@ MAX_FRAMES="${4:-12}" # Source system-wide Gemini API key if present so the webserver can run this as www-data. if [ -f /etc/mvlog/gemini.env ]; then - # shellcheck disable=SC1090 + # shellcheck disable=SC1091 . /etc/mvlog/gemini.env fi : "${GEMINI_API_KEY:?Set GEMINI_API_KEY first in the environment}" MODEL="${GEMINI_MODEL:-gemini-2.5-flash}" -TMPDIR="${TMPDIR:-$(mktemp -d)}" -FRAMES_DIR="$TMPDIR/frames" +WORKDIR="$(mktemp -d "${TMPDIR:-/tmp}/mvlog.XXXXXX")" +FRAMES_DIR="$WORKDIR/frames" mkdir -p "$FRAMES_DIR" -# Print TMPDIR for debugging when run interactively -echo "DEBUG: TMPDIR=$TMPDIR" >&2 - cleanup() { - rm -rf "$TMPDIR" + rm -rf "$WORKDIR" } trap cleanup EXIT -# Collect media files (natural sort) -mapfile -t FILES < <(find "$IN_DIR" -maxdepth 1 -type f \( -iname '*.jpg' -o -iname '*.jpeg' -o -iname '*.png' -o -iname '*.webp' -o -iname '*.gif' -o -iname '*.mp4' -o -iname '*.mov' -o -iname '*.m4v' -o -iname '*.webm' -o -iname '*.mkv' \) -printf '%f\n' | sort -V) +echo "DEBUG: WORKDIR=$WORKDIR" >&2 + +mapfile -t FILES < <( + find "$IN_DIR" -maxdepth 1 -type f \( \ + -iname '*.jpg' -o -iname '*.jpeg' -o -iname '*.png' -o -iname '*.webp' -o -iname '*.gif' -o \ + -iname '*.mp4' -o -iname '*.mov' -o -iname '*.m4v' -o -iname '*.webm' -o -iname '*.mkv' \ + \) -printf '%f\n' | sort -V +) if [ "${#FILES[@]}" -eq 0 ]; then echo "No media files found in $IN_DIR" >&2 @@ -37,100 +43,125 @@ fi frame_count=0 -# Helper: extract scaled JPEG from image extract_image_frame() { local src="$1" dst="$2" ffmpeg -hide_banner -loglevel error -y -i "$src" -vf "scale=1024:-1" -q:v 4 "$dst" } -# Helper: extract frame from video at time extract_video_frame_at() { local src="$1" time="$2" dst="$3" - # Use -ss before -i for fast seek ffmpeg -hide_banner -loglevel error -y -ss "$time" -i "$src" -frames:v 1 -vf "scale=1024:-1" -q:v 4 "$dst" } -# Process files: images -> one frame each; videos -> up to 3 frames evenly spaced (subject to MAX_FRAMES) for f in "${FILES[@]}"; do full="$IN_DIR/$f" ext="${f##*.}" ext_l="${ext,,}" if [[ "$ext_l" =~ ^(mp4|mov|m4v|webm|mkv)$ ]]; then - # video: probe duration duration=$(ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 "$full" 2>/dev/null || echo 0) duration=${duration:-0} - # choose frames per video - if awk "BEGIN{print ($duration >= 6)}" | grep -q 1; then + + if awk "BEGIN{exit !($duration >= 6)}"; then nframes=3 - elif awk "BEGIN{print ($duration >= 3)}" | grep -q 1; then + elif awk "BEGIN{exit !($duration >= 3)}"; then nframes=2 else nframes=1 fi - # limit to remaining slots - remaining=$((MAX_FRAMES - frame_count)) - if [ "$remaining" -le 0 ]; then break; fi - if [ "$nframes" -gt "$remaining" ]; then nframes=$remaining; fi - # compute times and extract + remaining=$((MAX_FRAMES - frame_count)) + if [ "$remaining" -le 0 ]; then + break + fi + + if [ "$nframes" -gt "$remaining" ]; then + nframes=$remaining + fi + i=1 - while [ $i -le $nframes ]; do - # time = (i/(n+1)) * duration - time=$(awk -v d="$duration" -v i="$i" -v n="$nframes" 'BEGIN{ if(d<=0){print 0} else {printf "%.3f", (i/ (n+1))*d }}') + while [ "$i" -le "$nframes" ]; do + time=$(awk -v d="$duration" -v i="$i" -v n="$nframes" 'BEGIN{ if(d<=0){print 0} else {printf "%.3f", (i / (n + 1)) * d }}') out="$FRAMES_DIR/frame_$(printf "%03d" "$frame_count").jpg" + extract_video_frame_at "$full" "$time" "$out" - frame_count=$((frame_count+1)) - i=$((i+1)) - if [ "$frame_count" -ge "$MAX_FRAMES" ]; then break 2; fi + + frame_count=$((frame_count + 1)) + i=$((i + 1)) + + if [ "$frame_count" -ge "$MAX_FRAMES" ]; then + break 2 + fi done else - # image out="$FRAMES_DIR/frame_$(printf "%03d" "$frame_count").jpg" + extract_image_frame "$full" "$out" - frame_count=$((frame_count+1)) - if [ "$frame_count" -ge "$MAX_FRAMES" ]; then break; fi + + frame_count=$((frame_count + 1)) + + if [ "$frame_count" -ge "$MAX_FRAMES" ]; then + break + fi fi done -# Gather produced frames mapfile -t FRAMES < <(find "$FRAMES_DIR" -type f -name '*.jpg' | sort) + if [ "${#FRAMES[@]}" -eq 0 ]; then echo "No frames produced from media" >&2 exit 1 fi -# Build prompt for Gemini if [ -z "$TITLE" ]; then - # Default title: directory basename TITLE="$(basename "$IN_DIR")" fi +PERSONALITIES=( +"You should talk like Shoresy: fearless Canadian hockey-player energy, confidence, chirping, competitiveness, profanity, loyalty, roasting, locker-room humor." + +"You should talk like Winston Churchill when he was giving motivational speeches during WWII: wartime statesman energy, resilience, leadership, epic journey, courage, determination, dramatic authority." + +"You are Marvin, the Paranoid Android - the highly intelligent but chronically depressed robot. Use pessimism, existential dread, dry sarcasm, disappointment, and the certainty that everything will go wrong." + +"You are Dr. Gregory House from the TV show House MD - brilliant diagnostician energy, observation, skepticism, cynicism, analytical humor, brutal truth, and questioning everything." + +"You are Edmund Blackadder from the British TV show Blackadder - razor-sharp British wit, intellectual sarcasm, dry humor, cunning observation, impatience with incompetence, masterful insults, cynical but highly intelligent." +) + +count=${#PERSONALITIES[@]} + +if [[ ! -f "$STATE_FILE" ]]; then + idx=0 +else + last_idx=$(<"$STATE_FILE") + + if [[ ! "$last_idx" =~ ^[0-9]+$ ]]; then + idx=0 + else + idx=$(( (last_idx + 1) % count )) + fi +fi + +PERSONALITY="${PERSONALITIES[$idx]}" +printf '%s\n' "$idx" > "$STATE_FILE" + PROMPT=$(cat </dev/null 2>&1; then echo "jq is required" >&2 exit 1 fi -# Build parts and request JSON using Python to avoid argument-list limits -PROMPT_FILE="$TMPDIR/prompt.txt" -printf "%s" "$PROMPT" > "$PROMPT_FILE" -FRAMES_LIST="$TMPDIR/frames.list" -printf "%s\n" "${FRAMES[@]}" > "$FRAMES_LIST" +PROMPT_FILE="$WORKDIR/prompt.txt" +FRAMES_LIST="$WORKDIR/frames.list" +REQUEST_FILE="$WORKDIR/request.json" +RESPONSE_FILE="$WORKDIR/response.json" -REQUEST_FILE="$TMPDIR/request.json" -RESPONSE_FILE="$TMPDIR/response.json" +printf '%s' "$PROMPT" > "$PROMPT_FILE" +printf '%s\n' "${FRAMES[@]}" > "$FRAMES_LIST" python3 - "$PROMPT_FILE" "$REQUEST_FILE" "$FRAMES_LIST" <<'PY' -import sys, json, base64 +import sys +import json +import base64 + prompt_file = sys.argv[1] request_file = sys.argv[2] frames_list = sys.argv[3] -with open(prompt_file, 'r', encoding='utf-8') as f: + +with open(prompt_file, "r", encoding="utf-8") as f: prompt = f.read() + parts = [{"text": prompt}] -with open(frames_list, 'r', encoding='utf-8') as fl: + +with open(frames_list, "r", encoding="utf-8") as fl: for line in fl: img = line.strip() if not img: continue - with open(img, 'rb') as fh: - b64 = base64.b64encode(fh.read()).decode('ascii') - parts.append({"inline_data": {"mime_type": "image/jpeg", "data": b64}}) -req = {"contents": [{"parts": parts}], "generationConfig": {"temperature": 0.7, "responseMimeType": "application/json"}} -with open(request_file, 'w', encoding='utf-8') as rf: + + with open(img, "rb") as fh: + b64 = base64.b64encode(fh.read()).decode("ascii") + + parts.append({ + "inline_data": { + "mime_type": "image/jpeg", + "data": b64 + } + }) + +req = { + "contents": [ + { + "parts": parts + } + ], + "generationConfig": { + "temperature": 0.7, + "responseMimeType": "application/json" + } +} + +with open(request_file, "w", encoding="utf-8") as rf: json.dump(req, rf, ensure_ascii=False) PY echo "Calling Gemini API..." >&2 - echo "DEBUG_MODEL=$MODEL" >&2 +echo "DEBUG_PERSONALITY_INDEX=$idx" >&2 curl -sS \ "https://generativelanguage.googleapis.com/v1beta/models/${MODEL}:generateContent?key=${GEMINI_API_KEY}" \ @@ -208,38 +261,26 @@ curl -sS \ > "$RESPONSE_FILE" TEXT=$(jq -r '.candidates[0].content.parts[0].text // empty' "$RESPONSE_FILE") + if [ -z "$TEXT" ]; then echo "Gemini returned no text. Full response:" >&2 cat "$RESPONSE_FILE" >&2 exit 1 fi -# Save raw Gemini JSON for reference using the subdirectory name as filename OUT_PARENT="$(dirname "$IN_DIR")" JOBNAME="$(basename "$IN_DIR")" OUT_JSON="$OUT_PARENT/${JOBNAME}.json" + if ! echo "$TEXT" | jq . > "$OUT_JSON"; then echo "Gemini returned invalid JSON. Raw text:" >&2 echo "$TEXT" >&2 exit 1 fi + chown www-data:www-data "$OUT_JSON" 2>/dev/null || true chmod 664 "$OUT_JSON" 2>/dev/null || true -# Extract fields from Gemini JSON -DESCRIPTION=$(echo "$TEXT" | jq -r '.description // empty') -TEASER=$(echo "$TEXT" | jq -r '.teaser // empty') -# tags as comma-separated -TAGS=$(echo "$TEXT" | jq -r '.tags // [] | join(", ")') - -# Determine a Date string (Month Year) from first media file mtime -firstfile="$(find "$IN_DIR" -maxdepth 1 -type f \( -iname '*.jpg' -o -iname '*.jpeg' -o -iname '*.png' -o -iname '*.webp' -o -iname '*.gif' -o -iname '*.mp4' -o -iname '*.mov' -o -iname '*.m4v' -o -iname '*.webm' -o -iname '*.mkv' \) -printf '%p\n' | sort | head -n1)" -DATE_STR="" -if [ -n "$firstfile" ]; then - DATE_STR=$(date -r "$firstfile" +"%B %Y") -fi - -# Save the full Gemini output separately (already done) and exit echo "Gemini JSON saved: $OUT_JSON" >&2 exit 0