Add remaining relevant files (auto)
This commit is contained in:
hbrain
parent
c2d64cb00a
commit
6b00b0d7fe
7 changed files with 649 additions and 1 deletions
242
bin/debug_generate_data.sh
Executable file
242
bin/debug_generate_data.sh
Executable file
|
|
@ -0,0 +1,242 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
IN_DIR="${1:?Usage: $0 <input-dir> [\"Title\"] [\"Location\"] [max_frames]}"
|
||||
TITLE="${2:-}"
|
||||
LOCATION="${3:-}"
|
||||
MAX_FRAMES="${4:-12}"
|
||||
|
||||
# Source system-wide Gemini API key if present so the webserver can run this as www-data.
|
||||
if [ -f /etc/mvlog/gemini.env ]; then
|
||||
# shellcheck disable=SC1090
|
||||
. /etc/mvlog/gemini.env
|
||||
fi
|
||||
|
||||
: "${GEMINI_API_KEY:?Set GEMINI_API_KEY first in the environment}"
|
||||
MODEL="${GEMINI_MODEL:-gemini-2.5-flash}"
|
||||
|
||||
TMPDIR="${TMPDIR:-$(mktemp -d)}"
|
||||
FRAMES_DIR="$TMPDIR/frames"
|
||||
mkdir -p "$FRAMES_DIR"
|
||||
|
||||
# Print TMPDIR for debugging when run interactively
|
||||
echo "DEBUG: TMPDIR=$TMPDIR" >&2
|
||||
|
||||
cleanup() {
|
||||
rm -rf "$TMPDIR"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# Collect media files (natural sort)
|
||||
mapfile -t FILES < <(find "$IN_DIR" -maxdepth 1 -type f \( -iname '*.jpg' -o -iname '*.jpeg' -o -iname '*.png' -o -iname '*.webp' -o -iname '*.gif' -o -iname '*.mp4' -o -iname '*.mov' -o -iname '*.m4v' -o -iname '*.webm' -o -iname '*.mkv' \) -printf '%f\n' | sort -V)
|
||||
|
||||
if [ "${#FILES[@]}" -eq 0 ]; then
|
||||
echo "No media files found in $IN_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
frame_count=0
|
||||
|
||||
# Helper: extract scaled JPEG from image
|
||||
extract_image_frame() {
|
||||
local src="$1" dst="$2"
|
||||
ffmpeg -hide_banner -loglevel error -y -i "$src" -vf "scale=1024:-1" -q:v 4 "$dst"
|
||||
}
|
||||
|
||||
# Helper: extract frame from video at time
|
||||
extract_video_frame_at() {
|
||||
local src="$1" time="$2" dst="$3"
|
||||
# Use -ss before -i for fast seek
|
||||
ffmpeg -hide_banner -loglevel error -y -ss "$time" -i "$src" -frames:v 1 -vf "scale=1024:-1" -q:v 4 "$dst"
|
||||
}
|
||||
|
||||
# Process files: images -> one frame each; videos -> up to 3 frames evenly spaced (subject to MAX_FRAMES)
|
||||
for f in "${FILES[@]}"; do
|
||||
full="$IN_DIR/$f"
|
||||
ext="${f##*.}"
|
||||
ext_l="${ext,,}"
|
||||
|
||||
if [[ "$ext_l" =~ ^(mp4|mov|m4v|webm|mkv)$ ]]; then
|
||||
# video: probe duration
|
||||
duration=$(ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 "$full" 2>/dev/null || echo 0)
|
||||
duration=${duration:-0}
|
||||
# choose frames per video
|
||||
if awk "BEGIN{print ($duration >= 6)}" | grep -q 1; then
|
||||
nframes=3
|
||||
elif awk "BEGIN{print ($duration >= 3)}" | grep -q 1; then
|
||||
nframes=2
|
||||
else
|
||||
nframes=1
|
||||
fi
|
||||
# limit to remaining slots
|
||||
remaining=$((MAX_FRAMES - frame_count))
|
||||
if [ "$remaining" -le 0 ]; then break; fi
|
||||
if [ "$nframes" -gt "$remaining" ]; then nframes=$remaining; fi
|
||||
|
||||
# compute times and extract
|
||||
i=1
|
||||
while [ $i -le $nframes ]; do
|
||||
# time = (i/(n+1)) * duration
|
||||
time=$(awk -v d="$duration" -v i="$i" -v n="$nframes" 'BEGIN{ if(d<=0){print 0} else {printf "%.3f", (i/ (n+1))*d }}')
|
||||
out="$FRAMES_DIR/frame_$(printf "%03d" "$frame_count").jpg"
|
||||
extract_video_frame_at "$full" "$time" "$out"
|
||||
frame_count=$((frame_count+1))
|
||||
i=$((i+1))
|
||||
if [ "$frame_count" -ge "$MAX_FRAMES" ]; then break 2; fi
|
||||
done
|
||||
else
|
||||
# image
|
||||
out="$FRAMES_DIR/frame_$(printf "%03d" "$frame_count").jpg"
|
||||
extract_image_frame "$full" "$out"
|
||||
frame_count=$((frame_count+1))
|
||||
if [ "$frame_count" -ge "$MAX_FRAMES" ]; then break; fi
|
||||
fi
|
||||
done
|
||||
|
||||
# Gather produced frames
|
||||
mapfile -t FRAMES < <(find "$FRAMES_DIR" -type f -name '*.jpg' | sort)
|
||||
if [ "${#FRAMES[@]}" -eq 0 ]; then
|
||||
echo "No frames produced from media" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Build prompt for Gemini
|
||||
if [ -z "$TITLE" ]; then
|
||||
# Default title: directory basename
|
||||
TITLE="$(basename "$IN_DIR")"
|
||||
fi
|
||||
|
||||
PROMPT=$(cat <<EOF
|
||||
You write short descriptions for my motorcycle/travel blog.
|
||||
You should talk like the main character from TV show Shoresy - profane, passionate,
|
||||
and hilariously obsessive titular protagonist of the acclaimed Canadian ice hockey comedy series.
|
||||
Use real quotes from the show as much as possible.
|
||||
|
||||
Title: "$TITLE"
|
||||
Location: "$LOCATION"
|
||||
|
||||
Look at these captured frames and write JSON only.
|
||||
|
||||
Required JSON format:
|
||||
{
|
||||
"description": "2-3 relaxed human sentences",
|
||||
"teaser": "one short teaser sentence",
|
||||
"tags": ["tag1", "tag2", "tag3", "tag4", "tag5"]
|
||||
}
|
||||
|
||||
Style:
|
||||
personal, natural, slightly casual, not corporate, not clickbait.
|
||||
Do not invent facts that are not visible in the images or provided in title/location.
|
||||
EOF
|
||||
)
|
||||
|
||||
PARTS_FILE="$TMPDIR/parts.json"
|
||||
|
||||
# Create parts with the prompt
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
echo "jq is required" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Build parts and request JSON using Python to avoid argument-list limits
|
||||
PROMPT_FILE="$TMPDIR/prompt.txt"
|
||||
printf "%s" "$PROMPT" > "$PROMPT_FILE"
|
||||
FRAMES_LIST="$TMPDIR/frames.list"
|
||||
printf "%s\n" "${FRAMES[@]}" > "$FRAMES_LIST"
|
||||
|
||||
REQUEST_FILE="$TMPDIR/request.json"
|
||||
RESPONSE_FILE="$TMPDIR/response.json"
|
||||
|
||||
python3 - "$PROMPT_FILE" "$REQUEST_FILE" "$FRAMES_LIST" <<'PY'
|
||||
import sys, json, base64
|
||||
prompt_file = sys.argv[1]
|
||||
request_file = sys.argv[2]
|
||||
frames_list = sys.argv[3]
|
||||
with open(prompt_file, 'r', encoding='utf-8') as f:
|
||||
prompt = f.read()
|
||||
parts = [{"text": prompt}]
|
||||
with open(frames_list, 'r', encoding='utf-8') as fl:
|
||||
for line in fl:
|
||||
img = line.strip()
|
||||
if not img:
|
||||
continue
|
||||
with open(img, 'rb') as fh:
|
||||
b64 = base64.b64encode(fh.read()).decode('ascii')
|
||||
parts.append({"inline_data": {"mime_type": "image/jpeg", "data": b64}})
|
||||
req = {"contents": [{"parts": parts}], "generationConfig": {"temperature": 0.7, "responseMimeType": "application/json"}}
|
||||
with open(request_file, 'w', encoding='utf-8') as rf:
|
||||
json.dump(req, rf, ensure_ascii=False)
|
||||
PY
|
||||
|
||||
echo "Calling Gemini API..."
|
||||
echo "DEBUG_MODEL=$MODEL" >&2
|
||||
|
||||
curl -sS \
|
||||
"https://generativelanguage.googleapis.com/v1beta/models/${MODEL}:generateContent?key=${GEMINI_API_KEY}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-X POST \
|
||||
-d @"$REQUEST_FILE" \
|
||||
> "$RESPONSE_FILE"
|
||||
# debug: save request/response for inspection
|
||||
cp "$REQUEST_FILE" "$IN_DIR/debug_request.json" || true
|
||||
cp "$RESPONSE_FILE" "$IN_DIR/debug_response.json" || true
|
||||
chown www-data:www-data "$IN_DIR/debug_request.json" "$IN_DIR/debug_response.json" 2>/dev/null || true
|
||||
|
||||
TEXT=$(jq -r '.candidates[0].content.parts[0].text // empty' "$RESPONSE_FILE")
|
||||
if [ -z "$TEXT" ]; then
|
||||
echo "Gemini returned no text. Full response:" >&2
|
||||
cat "$RESPONSE_FILE" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Save raw Gemini JSON for reference
|
||||
OUT_JSON="$IN_DIR/gemini_generated.json"
|
||||
echo "$TEXT" | jq . > "$OUT_JSON"
|
||||
|
||||
# Extract fields from Gemini JSON
|
||||
DESCRIPTION=$(echo "$TEXT" | jq -r '.description // empty')
|
||||
TEASER=$(echo "$TEXT" | jq -r '.teaser // empty')
|
||||
# tags as comma-separated
|
||||
TAGS=$(echo "$TEXT" | jq -r '.tags // [] | join(", ")')
|
||||
|
||||
# Determine a Date string (Month Year) from first media file mtime
|
||||
firstfile="$(find "$IN_DIR" -maxdepth 1 -type f \( -iname '*.jpg' -o -iname '*.jpeg' -o -iname '*.png' -o -iname '*.webp' -o -iname '*.gif' -o -iname '*.mp4' -o -iname '*.mov' -o -iname '*.m4v' -o -iname '*.webm' -o -iname '*.mkv' \) -printf '%p\n' | sort | head -n1)"
|
||||
DATE_STR=""
|
||||
if [ -n "$firstfile" ]; then
|
||||
DATE_STR=$(date -r "$firstfile" +"%B %Y")
|
||||
fi
|
||||
|
||||
# Update data.txt: only add a Description block if none exists already. Do NOT modify other fields.
|
||||
DATA_FILE="$IN_DIR/data.txt"
|
||||
# If Gemini provided no explicit 'description' field, fall back to the full response text.
|
||||
if [ -z "$DESCRIPTION" ]; then
|
||||
DESCRIPTION="$TEXT"
|
||||
fi
|
||||
|
||||
if [ -f "$DATA_FILE" ]; then
|
||||
if grep -qi '^[[:space:]]*description[[:space:]]*:' "$DATA_FILE"; then
|
||||
echo "data.txt already contains a Description; leaving unchanged" >&2
|
||||
else
|
||||
{
|
||||
echo ""
|
||||
echo "Description: |"
|
||||
printf '%s\n' "$DESCRIPTION" | sed 's/^/ /'
|
||||
} >> "$DATA_FILE"
|
||||
chown www-data:www-data "$DATA_FILE" 2>/dev/null || true
|
||||
chmod 664 "$DATA_FILE" 2>/dev/null || true
|
||||
echo "Appended Description to $DATA_FILE" >&2
|
||||
fi
|
||||
else
|
||||
{
|
||||
echo "Description: |"
|
||||
printf '%s\n' "$DESCRIPTION" | sed 's/^/ /'
|
||||
} > "$DATA_FILE"
|
||||
chown www-data:www-data "$DATA_FILE" 2>/dev/null || true
|
||||
chmod 664 "$DATA_FILE" 2>/dev/null || true
|
||||
echo "Created $DATA_FILE with Description" >&2
|
||||
fi
|
||||
|
||||
# Save the full Gemini output separately (already done) and exit
|
||||
echo "Gemini JSON saved: $OUT_JSON"
|
||||
|
||||
exit 0
|
||||
|
|
@ -108,7 +108,7 @@ if [ -z "$TITLE" ]; then
|
|||
fi
|
||||
|
||||
PROMPT=$(cat <<EOF
|
||||
You write short descriptions for my motorcycle/travel blog.
|
||||
You write descriptions (not les tha 60 or more than 70 words) for my motorcycle/travel blog.
|
||||
You should talk like the main character from TV show Shoresy - profane, passionate,
|
||||
and hilariously obsessive titular protagonist of the acclaimed Canadian ice hockey comedy series.
|
||||
Use real quotes from the show as much as possible.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue