- Stops using openrouter/auto by default; overrides it to openai/gpt-4o-mini unless OPENROUTER_ALLOW_AUTO=1.

- Retries OpenRouter up to 3 times if response is empty or malformed.
 - Validates that JSON has required fields before saving.
 - Saves last raw OpenRouter response/text for debugging:
     - .mvlog-last-openrouter-response.json
     - .mvlog-last-openrouter-text.txt
 - Only advances .personality_index after valid JSON is saved.

bin/generate_data.sh

@@ -20,7 +20,12 @@ if [ -f /etc/mvlog/openrouter.env ]; then
 fi
 
 : "${OPENROUTER_API_KEY:?Set OPENROUTER_API_KEY first in the environment}"
-MODEL="${OPENROUTER_MODEL:-openrouter/auto}"
+MODEL="${OPENROUTER_MODEL:-openai/gpt-4o-mini}"
+if [ "$MODEL" = "openrouter/auto" ] && [ "${OPENROUTER_ALLOW_AUTO:-}" != "1" ]; then
+  echo "WARN: openrouter/auto is unreliable for strict JSON; using openai/gpt-4o-mini. Set OPENROUTER_MODEL to a specific vision model, or OPENROUTER_ALLOW_AUTO=1 to force auto." >&2
+  MODEL="openai/gpt-4o-mini"
+fi
+OPENROUTER_MAX_RETRIES="${OPENROUTER_MAX_RETRIES:-3}"
 OPENROUTER_SITE_URL="${OPENROUTER_SITE_URL:-https://bubulescu.org}"
 OPENROUTER_SITE_NAME="${OPENROUTER_SITE_NAME:-MVLog}"
 
@@ -169,7 +174,6 @@ fi
 
 PERSONALITY_KEY="${PERSONALITY_KEYS[$idx]}"
 PERSONALITY="${PERSONALITY_TEXTS[$idx]}"
-printf '%s\n' "$idx" > "$STATE_FILE"
 
 PROMPT=$(cat <<EOF
 Look at these captured frames from a motorcycle trip.
@@ -297,11 +301,24 @@ req = {
 with open(request_file, "w", encoding="utf-8") as rf:
     json.dump(req, rf, ensure_ascii=False)
 PY
-echo "Calling OpenRouter API..." >&2
+OUT_PARENT="$(dirname "$IN_DIR")"
-echo "DEBUG_MODEL=$MODEL" >&2
+JOBNAME="$(basename "$IN_DIR")"
-echo "DEBUG_PERSONALITY_INDEX=$idx" >&2
+OUT_JSON="$OUT_PARENT/${JOBNAME}.json"
+OUT_TMP="$OUT_JSON.tmp.$$"
+DEBUG_RESPONSE_FILE="$IN_DIR/.mvlog-last-openrouter-response.json"
+DEBUG_TEXT_FILE="$IN_DIR/.mvlog-last-openrouter-text.txt"
 
-curl -sS \
+if [[ ! "$OPENROUTER_MAX_RETRIES" =~ ^[0-9]+$ ]] || [ "$OPENROUTER_MAX_RETRIES" -lt 1 ]; then
+  OPENROUTER_MAX_RETRIES=3
+fi
+
+attempt=1
+while [ "$attempt" -le "$OPENROUTER_MAX_RETRIES" ]; do
+  echo "Calling OpenRouter API (attempt $attempt/$OPENROUTER_MAX_RETRIES)..." >&2
+  echo "DEBUG_MODEL=$MODEL" >&2
+  echo "DEBUG_PERSONALITY_INDEX=$idx" >&2
+
+  if ! curl -sS \
     "https://openrouter.ai/api/v1/chat/completions" \
     -H "Authorization: Bearer ${OPENROUTER_API_KEY}" \
     -H "Content-Type: application/json" \
@@ -309,29 +326,56 @@ curl -sS \
     -H "X-Title: ${OPENROUTER_SITE_NAME}" \
     -X POST \
     -d @"$REQUEST_FILE" \
-  > "$RESPONSE_FILE"
+    > "$RESPONSE_FILE"; then
+    echo "OpenRouter curl request failed on attempt $attempt" >&2
+    cp "$RESPONSE_FILE" "$DEBUG_RESPONSE_FILE" 2>/dev/null || true
+    attempt=$((attempt + 1))
+    sleep 2
+    continue
+  fi
 
-TEXT=$(jq -r '.choices[0].message.content // empty' "$RESPONSE_FILE")
+  cp "$RESPONSE_FILE" "$DEBUG_RESPONSE_FILE" 2>/dev/null || true
+  chown www-data:www-data "$DEBUG_RESPONSE_FILE" 2>/dev/null || true
+  chmod 664 "$DEBUG_RESPONSE_FILE" 2>/dev/null || true
 
-if [ -z "$TEXT" ]; then
+  TEXT=$(jq -r '.choices[0].message.content // empty' "$RESPONSE_FILE" 2>/dev/null || true)
-  echo "OpenRouter returned no text. Full response:" >&2
+
+  if [ -z "$TEXT" ]; then
+    api_error=$(jq -r '.error.message // empty' "$RESPONSE_FILE" 2>/dev/null || true)
+    if [ -n "$api_error" ]; then
+      echo "OpenRouter returned no text on attempt $attempt: $api_error" >&2
+    else
+      echo "OpenRouter returned no text on attempt $attempt. Full response:" >&2
       cat "$RESPONSE_FILE" >&2
-  exit 1
+    fi
-fi
+    attempt=$((attempt + 1))
+    sleep 2
+    continue
+  fi
 
-OUT_PARENT="$(dirname "$IN_DIR")"
+  printf '%s\n' "$TEXT" > "$DEBUG_TEXT_FILE" 2>/dev/null || true
-JOBNAME="$(basename "$IN_DIR")"
+  chown www-data:www-data "$DEBUG_TEXT_FILE" 2>/dev/null || true
-OUT_JSON="$OUT_PARENT/${JOBNAME}.json"
+  chmod 664 "$DEBUG_TEXT_FILE" 2>/dev/null || true
 
-if ! echo "$TEXT" | jq . > "$OUT_JSON"; then
+  if printf '%s' "$TEXT" | jq -e 'type == "object" and (.personality | type == "string") and (.quote_da | type == "string") and (.teaser | type == "string") and (.description | type == "string") and (.tags | type == "array")' >/dev/null \
-  echo "OpenRouter returned invalid JSON. Raw text:" >&2
+    && printf '%s' "$TEXT" | jq . > "$OUT_TMP"; then
-  echo "$TEXT" >&2
+    mv "$OUT_TMP" "$OUT_JSON"
-  exit 1
+    chown www-data:www-data "$OUT_JSON" 2>/dev/null || true
-fi
+    chmod 664 "$OUT_JSON" 2>/dev/null || true
+    printf '%s\n' "$idx" > "$STATE_FILE"
+    chown www-data:www-data "$STATE_FILE" 2>/dev/null || true
+    chmod 664 "$STATE_FILE" 2>/dev/null || true
+    echo "OpenRouter JSON saved: $OUT_JSON" >&2
+    exit 0
+  fi
 
-chown www-data:www-data "$OUT_JSON" 2>/dev/null || true
+  rm -f "$OUT_TMP"
-chmod 664 "$OUT_JSON" 2>/dev/null || true
+  echo "OpenRouter returned invalid JSON on attempt $attempt. Raw text saved to $DEBUG_TEXT_FILE" >&2
+  printf '%s\n' "$TEXT" >&2
+  attempt=$((attempt + 1))
+  sleep 2
+done
 
-echo "OpenRouter JSON saved: $OUT_JSON" >&2
+rm -f "$OUT_TMP"
-
+echo "OpenRouter did not return valid JSON after $OPENROUTER_MAX_RETRIES attempt(s). Last raw response: $DEBUG_RESPONSE_FILE; last text: $DEBUG_TEXT_FILE" >&2
-exit 0
+exit 1