Keep MVLog render temp files in work directory

deploy/systemd/mvlog-worker.service

@@ -6,7 +6,11 @@ After=network-online.target
 [Service]
 Type=oneshot
 WorkingDirectory=/srv/mvlog/app
-ExecStart=/srv/mvlog/app/mvlog_worker.py --local --remote-root /var/www/html --work-dir /srv/mvlog/work
+Environment=TMPDIR=/srv/mvlog/work/tmp
+Environment=TEMP=/srv/mvlog/work/tmp
+Environment=TMP=/srv/mvlog/work/tmp
+Environment=MOVMAKER_TMPDIR=/srv/mvlog/work/tmp
+ExecStart=/srv/mvlog/app/mvlog_worker.py --local --remote-root /var/www/html --work-dir /srv/mvlog/work --tmp-dir /srv/mvlog/work/tmp
 User=hbrain
 Group=www-data
 UMask=0002

movmaker.py

@@ -1334,6 +1334,7 @@ def main(argv: list[str] | None = None) -> int:
     parser.add_argument("--preset", default="veryfast", help="x264 speed preset, e.g. ultrafast, superfast, veryfast")
     parser.add_argument("--crf", type=int, default=20, help="x264 quality; lower is better/larger, higher is faster/smaller")
     parser.add_argument("--jobs", type=int, default=1, help="parallel clip preparation jobs; try 2 on Raspberry Pi")
+    parser.add_argument("--tmp-dir", type=Path, default=Path(os.environ["MOVMAKER_TMPDIR"]) if os.environ.get("MOVMAKER_TMPDIR") else None, help="directory for large temporary render files; defaults to TMPDIR/tempfile")
     parser.add_argument("--preview", action="store_true", help="fast preview: 1280x720, 24 fps, ultrafast, CRF 28")
     parser.add_argument("--music-genre", default="cinematic punk rock", help="Jamendo music search query used only when input has no audio files")
     args = parser.parse_args(argv)
@@ -1377,6 +1378,10 @@ def main(argv: list[str] | None = None) -> int:
             audio_files.append(downloaded_audio)
 
     output = default_output_path(media[0].path, data, args.out_dir)
+    temp_dir = args.tmp_dir
+    if temp_dir is not None:
+        temp_dir.mkdir(parents=True, exist_ok=True)
+        console(f"Temp dir: {temp_dir}")
 
     # Image duration means fully visible time, not including crossfades.
     # Middle images need both a fade-in and fade-out added to their clip length.
@@ -1402,7 +1407,7 @@ def main(argv: list[str] | None = None) -> int:
     if data.title or data.date or data.place:
         console("Title data: " + " | ".join(x for x in [data.title, data.date, data.place] if x))
 
-    with tempfile.TemporaryDirectory(prefix="movmaker-") as tmp_name:
+    with tempfile.TemporaryDirectory(prefix="movmaker-", dir=temp_dir) as tmp_name:
         tmp = Path(tmp_name)
         jobs = max(1, args.jobs)
         clips = [tmp / f"clip_{idx:04d}.mp4" for idx in range(len(media))]

mvlog_worker.py

@@ -62,13 +62,25 @@ def safe_chmod(path: Path, mode: int) -> None:
         pass
 
 
+def local_dir_accessible(path: Path) -> bool:
+    return path.is_dir() and os.access(path, os.R_OK | os.X_OK)
+
+
 def list_jobs(host: str | None, remote_root: str) -> list[str]:
     if host is None:
         root = Path(remote_root) / "in-dir"
         if not root.is_dir():
             return []
-        return sorted(path.name for path in root.iterdir() if path.is_dir() and not path.name.startswith('.'))
+        jobs = []
-    cmd = f"cd {remote_q(remote_root)} && find in-dir -mindepth 1 -maxdepth 1 -type d ! -name '.*' -printf '%f\\n' | sort"
+        for path in root.iterdir():
+            if path.name.startswith('.') or not path.is_dir():
+                continue
+            if local_dir_accessible(path):
+                jobs.append(path.name)
+            else:
+                print(f"skip inaccessible input dir: {path.name}")
+        return sorted(jobs)
+    cmd = f"cd {remote_q(remote_root)} && find in-dir -mindepth 1 -maxdepth 1 -type d ! -name '.*' -readable -executable -printf '%f\\n' | sort"
     out = ssh(host, cmd).stdout
     return [line.strip() for line in out.splitlines() if line.strip()]
 
@@ -305,7 +317,7 @@ def upload_audio_files(host: str | None, job_dir: str, local_input: Path) -> lis
     return uploaded
 
 
-def render_job(local_input: Path, local_out: Path, extra_args: list[str]) -> Path:
+def render_job(local_input: Path, local_out: Path, extra_args: list[str], temp_dir: Path) -> Path:
     local_out.mkdir(parents=True, exist_ok=True)
     intro_logo = INTRO_LOGO if INTRO_LOGO.exists() else None
     intro_args = []
@@ -314,7 +326,7 @@ def render_job(local_input: Path, local_out: Path, extra_args: list[str]) -> Pat
             "--intro-logo", str(intro_logo),
             "--intro-logo-alpha", INTRO_LOGO_ALPHA,
         ]
-    cmd = [sys.executable, str(MOVMAKER), str(local_input), "--out-dir", str(local_out), *intro_args, *extra_args]
+    cmd = [sys.executable, str(MOVMAKER), str(local_input), "--out-dir", str(local_out), "--tmp-dir", str(temp_dir), *intro_args, *extra_args]
     log_dir = local_input / ".logs"
     log_dir.mkdir(parents=True, exist_ok=True)
     log_path = log_dir / f"mvlog_worker_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log"
@@ -338,8 +350,11 @@ def render_job(local_input: Path, local_out: Path, extra_args: list[str]) -> Pat
     return outputs[0]
 
 
-def process_job(host: str | None, remote_root: str, job: str, work_dir: Path, extra_args: list[str], force: bool) -> None:
+def process_job(host: str | None, remote_root: str, job: str, work_dir: Path, temp_dir: Path, extra_args: list[str], force: bool) -> None:
     job_dir = f"{remote_root}/in-dir/{job}"
+    if host is None and not local_dir_accessible(Path(job_dir)):
+        print(f"skip inaccessible input dir: {job}")
+        return
     article_id = read_article_id(host, job_dir)
     state = read_state(host, job_dir)
     full_requested = remote_file_exists(host, f"{job_dir}/{ENABLED_NAME}")
@@ -390,7 +405,7 @@ def process_job(host: str | None, remote_root: str, job: str, work_dir: Path, ex
         copy_job_from_remote(host, job_dir, local_input)
         render_args = [*extra_args, "--preview"] if preview_mode else extra_args
         print(f"rendering {'preview: ' if preview_mode else ''}{job}")
-        local_mp4 = render_job(local_input, local_out, render_args)
+        local_mp4 = render_job(local_input, local_out, render_args, temp_dir)
         if preview_mode:
             preview_mp4 = local_mp4.with_name(f"{local_mp4.stem}_preview{local_mp4.suffix}")
             local_mp4.rename(preview_mp4)
@@ -495,6 +510,7 @@ def main() -> int:
     parser.add_argument("--host", default="192.168.0.204", help="SSH host for the web server")
     parser.add_argument("--remote-root", default="/var/www/html", help="Remote MVLog web root")
     parser.add_argument("--work-dir", default=str(Path.home() / ".cache" / "mvlog-worker"), help="Local work directory")
+    parser.add_argument("--tmp-dir", default=None, help="Large temporary render directory; defaults to WORK_DIR/tmp")
     parser.add_argument("--job", action="append", help="Only process this input dir name; may be repeated")
     parser.add_argument("--force", action="store_true", help="Render even if fingerprint is unchanged")
     parser.add_argument("--movmaker-arg", action="append", default=[], help="Extra argument passed to movmaker; may be repeated")
@@ -502,9 +518,15 @@ def main() -> int:
 
     work_dir = Path(args.work_dir)
     work_dir.mkdir(parents=True, exist_ok=True)
+    temp_dir = Path(args.tmp_dir) if args.tmp_dir else work_dir / "tmp"
+    temp_dir.mkdir(parents=True, exist_ok=True)
+    os.environ["TMPDIR"] = str(temp_dir)
+    os.environ["TEMP"] = str(temp_dir)
+    os.environ["TMP"] = str(temp_dir)
+    os.environ["MOVMAKER_TMPDIR"] = str(temp_dir)
     jobs = args.job or list_jobs(None if args.local else args.host, args.remote_root)
     for job in jobs:
-        process_job(None if args.local else args.host, args.remote_root, job, work_dir, args.movmaker_arg, args.force)
+        process_job(None if args.local else args.host, args.remote_root, job, work_dir, temp_dir, args.movmaker_arg, args.force)
     return 0