From be70760320a1ed6977726a7b0f5609a37a44cb38 Mon Sep 17 00:00:00 2001 From: hbrain Date: Fri, 19 Jun 2026 18:06:25 +0200 Subject: [PATCH] - Stops using openrouter/auto by default; overrides it to openai/gpt-4o-mini unless OPENROUTER_ALLOW_AUTO=1. - Retries OpenRouter up to 3 times if response is empty or malformed. - Validates that JSON has required fields before saving. - Saves last raw OpenRouter response/text for debugging: - .mvlog-last-openrouter-response.json - .mvlog-last-openrouter-text.txt - Only advances .personality_index after valid JSON is saved. --- bin/generate_data.sh | 108 ++++++++++++++++++++++++++++++------------- 1 file changed, 76 insertions(+), 32 deletions(-) diff --git a/bin/generate_data.sh b/bin/generate_data.sh index 53413d2..6107ebb 100755 --- a/bin/generate_data.sh +++ b/bin/generate_data.sh @@ -20,7 +20,12 @@ if [ -f /etc/mvlog/openrouter.env ]; then fi : "${OPENROUTER_API_KEY:?Set OPENROUTER_API_KEY first in the environment}" -MODEL="${OPENROUTER_MODEL:-openrouter/auto}" +MODEL="${OPENROUTER_MODEL:-openai/gpt-4o-mini}" +if [ "$MODEL" = "openrouter/auto" ] && [ "${OPENROUTER_ALLOW_AUTO:-}" != "1" ]; then + echo "WARN: openrouter/auto is unreliable for strict JSON; using openai/gpt-4o-mini. Set OPENROUTER_MODEL to a specific vision model, or OPENROUTER_ALLOW_AUTO=1 to force auto." >&2 + MODEL="openai/gpt-4o-mini" +fi +OPENROUTER_MAX_RETRIES="${OPENROUTER_MAX_RETRIES:-3}" OPENROUTER_SITE_URL="${OPENROUTER_SITE_URL:-https://bubulescu.org}" OPENROUTER_SITE_NAME="${OPENROUTER_SITE_NAME:-MVLog}" @@ -169,7 +174,6 @@ fi PERSONALITY_KEY="${PERSONALITY_KEYS[$idx]}" PERSONALITY="${PERSONALITY_TEXTS[$idx]}" -printf '%s\n' "$idx" > "$STATE_FILE" PROMPT=$(cat <&2 -echo "DEBUG_MODEL=$MODEL" >&2 -echo "DEBUG_PERSONALITY_INDEX=$idx" >&2 - -curl -sS \ - "https://openrouter.ai/api/v1/chat/completions" \ - -H "Authorization: Bearer ${OPENROUTER_API_KEY}" \ - -H "Content-Type: application/json" \ - -H "HTTP-Referer: ${OPENROUTER_SITE_URL}" \ - -H "X-Title: ${OPENROUTER_SITE_NAME}" \ - -X POST \ - -d @"$REQUEST_FILE" \ - > "$RESPONSE_FILE" - -TEXT=$(jq -r '.choices[0].message.content // empty' "$RESPONSE_FILE") - -if [ -z "$TEXT" ]; then - echo "OpenRouter returned no text. Full response:" >&2 - cat "$RESPONSE_FILE" >&2 - exit 1 -fi - OUT_PARENT="$(dirname "$IN_DIR")" JOBNAME="$(basename "$IN_DIR")" OUT_JSON="$OUT_PARENT/${JOBNAME}.json" +OUT_TMP="$OUT_JSON.tmp.$$" +DEBUG_RESPONSE_FILE="$IN_DIR/.mvlog-last-openrouter-response.json" +DEBUG_TEXT_FILE="$IN_DIR/.mvlog-last-openrouter-text.txt" -if ! echo "$TEXT" | jq . > "$OUT_JSON"; then - echo "OpenRouter returned invalid JSON. Raw text:" >&2 - echo "$TEXT" >&2 - exit 1 +if [[ ! "$OPENROUTER_MAX_RETRIES" =~ ^[0-9]+$ ]] || [ "$OPENROUTER_MAX_RETRIES" -lt 1 ]; then + OPENROUTER_MAX_RETRIES=3 fi -chown www-data:www-data "$OUT_JSON" 2>/dev/null || true -chmod 664 "$OUT_JSON" 2>/dev/null || true +attempt=1 +while [ "$attempt" -le "$OPENROUTER_MAX_RETRIES" ]; do + echo "Calling OpenRouter API (attempt $attempt/$OPENROUTER_MAX_RETRIES)..." >&2 + echo "DEBUG_MODEL=$MODEL" >&2 + echo "DEBUG_PERSONALITY_INDEX=$idx" >&2 -echo "OpenRouter JSON saved: $OUT_JSON" >&2 + if ! curl -sS \ + "https://openrouter.ai/api/v1/chat/completions" \ + -H "Authorization: Bearer ${OPENROUTER_API_KEY}" \ + -H "Content-Type: application/json" \ + -H "HTTP-Referer: ${OPENROUTER_SITE_URL}" \ + -H "X-Title: ${OPENROUTER_SITE_NAME}" \ + -X POST \ + -d @"$REQUEST_FILE" \ + > "$RESPONSE_FILE"; then + echo "OpenRouter curl request failed on attempt $attempt" >&2 + cp "$RESPONSE_FILE" "$DEBUG_RESPONSE_FILE" 2>/dev/null || true + attempt=$((attempt + 1)) + sleep 2 + continue + fi -exit 0 + cp "$RESPONSE_FILE" "$DEBUG_RESPONSE_FILE" 2>/dev/null || true + chown www-data:www-data "$DEBUG_RESPONSE_FILE" 2>/dev/null || true + chmod 664 "$DEBUG_RESPONSE_FILE" 2>/dev/null || true + + TEXT=$(jq -r '.choices[0].message.content // empty' "$RESPONSE_FILE" 2>/dev/null || true) + + if [ -z "$TEXT" ]; then + api_error=$(jq -r '.error.message // empty' "$RESPONSE_FILE" 2>/dev/null || true) + if [ -n "$api_error" ]; then + echo "OpenRouter returned no text on attempt $attempt: $api_error" >&2 + else + echo "OpenRouter returned no text on attempt $attempt. Full response:" >&2 + cat "$RESPONSE_FILE" >&2 + fi + attempt=$((attempt + 1)) + sleep 2 + continue + fi + + printf '%s\n' "$TEXT" > "$DEBUG_TEXT_FILE" 2>/dev/null || true + chown www-data:www-data "$DEBUG_TEXT_FILE" 2>/dev/null || true + chmod 664 "$DEBUG_TEXT_FILE" 2>/dev/null || true + + if printf '%s' "$TEXT" | jq -e 'type == "object" and (.personality | type == "string") and (.quote_da | type == "string") and (.teaser | type == "string") and (.description | type == "string") and (.tags | type == "array")' >/dev/null \ + && printf '%s' "$TEXT" | jq . > "$OUT_TMP"; then + mv "$OUT_TMP" "$OUT_JSON" + chown www-data:www-data "$OUT_JSON" 2>/dev/null || true + chmod 664 "$OUT_JSON" 2>/dev/null || true + printf '%s\n' "$idx" > "$STATE_FILE" + chown www-data:www-data "$STATE_FILE" 2>/dev/null || true + chmod 664 "$STATE_FILE" 2>/dev/null || true + echo "OpenRouter JSON saved: $OUT_JSON" >&2 + exit 0 + fi + + rm -f "$OUT_TMP" + echo "OpenRouter returned invalid JSON on attempt $attempt. Raw text saved to $DEBUG_TEXT_FILE" >&2 + printf '%s\n' "$TEXT" >&2 + attempt=$((attempt + 1)) + sleep 2 +done + +rm -f "$OUT_TMP" +echo "OpenRouter did not return valid JSON after $OPENROUTER_MAX_RETRIES attempt(s). Last raw response: $DEBUG_RESPONSE_FILE; last text: $DEBUG_TEXT_FILE" >&2 +exit 1