diff --git a/deploy/systemd/mvlog-worker.service b/deploy/systemd/mvlog-worker.service index fdf5909..d90a1e4 100644 --- a/deploy/systemd/mvlog-worker.service +++ b/deploy/systemd/mvlog-worker.service @@ -6,7 +6,11 @@ After=network-online.target [Service] Type=oneshot WorkingDirectory=/srv/mvlog/app -ExecStart=/srv/mvlog/app/mvlog_worker.py --local --remote-root /var/www/html --work-dir /srv/mvlog/work +Environment=TMPDIR=/srv/mvlog/work/tmp +Environment=TEMP=/srv/mvlog/work/tmp +Environment=TMP=/srv/mvlog/work/tmp +Environment=MOVMAKER_TMPDIR=/srv/mvlog/work/tmp +ExecStart=/srv/mvlog/app/mvlog_worker.py --local --remote-root /var/www/html --work-dir /srv/mvlog/work --tmp-dir /srv/mvlog/work/tmp User=hbrain Group=www-data UMask=0002 diff --git a/movmaker.py b/movmaker.py index 1946213..730db59 100755 --- a/movmaker.py +++ b/movmaker.py @@ -1334,6 +1334,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--preset", default="veryfast", help="x264 speed preset, e.g. ultrafast, superfast, veryfast") parser.add_argument("--crf", type=int, default=20, help="x264 quality; lower is better/larger, higher is faster/smaller") parser.add_argument("--jobs", type=int, default=1, help="parallel clip preparation jobs; try 2 on Raspberry Pi") + parser.add_argument("--tmp-dir", type=Path, default=Path(os.environ["MOVMAKER_TMPDIR"]) if os.environ.get("MOVMAKER_TMPDIR") else None, help="directory for large temporary render files; defaults to TMPDIR/tempfile") parser.add_argument("--preview", action="store_true", help="fast preview: 1280x720, 24 fps, ultrafast, CRF 28") parser.add_argument("--music-genre", default="cinematic punk rock", help="Jamendo music search query used only when input has no audio files") args = parser.parse_args(argv) @@ -1377,6 +1378,10 @@ def main(argv: list[str] | None = None) -> int: audio_files.append(downloaded_audio) output = default_output_path(media[0].path, data, args.out_dir) + temp_dir = args.tmp_dir + if temp_dir is not None: + temp_dir.mkdir(parents=True, exist_ok=True) + console(f"Temp dir: {temp_dir}") # Image duration means fully visible time, not including crossfades. # Middle images need both a fade-in and fade-out added to their clip length. @@ -1402,7 +1407,7 @@ def main(argv: list[str] | None = None) -> int: if data.title or data.date or data.place: console("Title data: " + " | ".join(x for x in [data.title, data.date, data.place] if x)) - with tempfile.TemporaryDirectory(prefix="movmaker-") as tmp_name: + with tempfile.TemporaryDirectory(prefix="movmaker-", dir=temp_dir) as tmp_name: tmp = Path(tmp_name) jobs = max(1, args.jobs) clips = [tmp / f"clip_{idx:04d}.mp4" for idx in range(len(media))] diff --git a/mvlog_worker.py b/mvlog_worker.py index 4ff9bdd..9175ca7 100755 --- a/mvlog_worker.py +++ b/mvlog_worker.py @@ -62,13 +62,25 @@ def safe_chmod(path: Path, mode: int) -> None: pass +def local_dir_accessible(path: Path) -> bool: + return path.is_dir() and os.access(path, os.R_OK | os.X_OK) + + def list_jobs(host: str | None, remote_root: str) -> list[str]: if host is None: root = Path(remote_root) / "in-dir" if not root.is_dir(): return [] - return sorted(path.name for path in root.iterdir() if path.is_dir() and not path.name.startswith('.')) - cmd = f"cd {remote_q(remote_root)} && find in-dir -mindepth 1 -maxdepth 1 -type d ! -name '.*' -printf '%f\\n' | sort" + jobs = [] + for path in root.iterdir(): + if path.name.startswith('.') or not path.is_dir(): + continue + if local_dir_accessible(path): + jobs.append(path.name) + else: + print(f"skip inaccessible input dir: {path.name}") + return sorted(jobs) + cmd = f"cd {remote_q(remote_root)} && find in-dir -mindepth 1 -maxdepth 1 -type d ! -name '.*' -readable -executable -printf '%f\\n' | sort" out = ssh(host, cmd).stdout return [line.strip() for line in out.splitlines() if line.strip()] @@ -305,7 +317,7 @@ def upload_audio_files(host: str | None, job_dir: str, local_input: Path) -> lis return uploaded -def render_job(local_input: Path, local_out: Path, extra_args: list[str]) -> Path: +def render_job(local_input: Path, local_out: Path, extra_args: list[str], temp_dir: Path) -> Path: local_out.mkdir(parents=True, exist_ok=True) intro_logo = INTRO_LOGO if INTRO_LOGO.exists() else None intro_args = [] @@ -314,7 +326,7 @@ def render_job(local_input: Path, local_out: Path, extra_args: list[str]) -> Pat "--intro-logo", str(intro_logo), "--intro-logo-alpha", INTRO_LOGO_ALPHA, ] - cmd = [sys.executable, str(MOVMAKER), str(local_input), "--out-dir", str(local_out), *intro_args, *extra_args] + cmd = [sys.executable, str(MOVMAKER), str(local_input), "--out-dir", str(local_out), "--tmp-dir", str(temp_dir), *intro_args, *extra_args] log_dir = local_input / ".logs" log_dir.mkdir(parents=True, exist_ok=True) log_path = log_dir / f"mvlog_worker_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log" @@ -338,8 +350,11 @@ def render_job(local_input: Path, local_out: Path, extra_args: list[str]) -> Pat return outputs[0] -def process_job(host: str | None, remote_root: str, job: str, work_dir: Path, extra_args: list[str], force: bool) -> None: +def process_job(host: str | None, remote_root: str, job: str, work_dir: Path, temp_dir: Path, extra_args: list[str], force: bool) -> None: job_dir = f"{remote_root}/in-dir/{job}" + if host is None and not local_dir_accessible(Path(job_dir)): + print(f"skip inaccessible input dir: {job}") + return article_id = read_article_id(host, job_dir) state = read_state(host, job_dir) full_requested = remote_file_exists(host, f"{job_dir}/{ENABLED_NAME}") @@ -390,7 +405,7 @@ def process_job(host: str | None, remote_root: str, job: str, work_dir: Path, ex copy_job_from_remote(host, job_dir, local_input) render_args = [*extra_args, "--preview"] if preview_mode else extra_args print(f"rendering {'preview: ' if preview_mode else ''}{job}") - local_mp4 = render_job(local_input, local_out, render_args) + local_mp4 = render_job(local_input, local_out, render_args, temp_dir) if preview_mode: preview_mp4 = local_mp4.with_name(f"{local_mp4.stem}_preview{local_mp4.suffix}") local_mp4.rename(preview_mp4) @@ -495,6 +510,7 @@ def main() -> int: parser.add_argument("--host", default="192.168.0.204", help="SSH host for the web server") parser.add_argument("--remote-root", default="/var/www/html", help="Remote MVLog web root") parser.add_argument("--work-dir", default=str(Path.home() / ".cache" / "mvlog-worker"), help="Local work directory") + parser.add_argument("--tmp-dir", default=None, help="Large temporary render directory; defaults to WORK_DIR/tmp") parser.add_argument("--job", action="append", help="Only process this input dir name; may be repeated") parser.add_argument("--force", action="store_true", help="Render even if fingerprint is unchanged") parser.add_argument("--movmaker-arg", action="append", default=[], help="Extra argument passed to movmaker; may be repeated") @@ -502,9 +518,15 @@ def main() -> int: work_dir = Path(args.work_dir) work_dir.mkdir(parents=True, exist_ok=True) + temp_dir = Path(args.tmp_dir) if args.tmp_dir else work_dir / "tmp" + temp_dir.mkdir(parents=True, exist_ok=True) + os.environ["TMPDIR"] = str(temp_dir) + os.environ["TEMP"] = str(temp_dir) + os.environ["TMP"] = str(temp_dir) + os.environ["MOVMAKER_TMPDIR"] = str(temp_dir) jobs = args.job or list_jobs(None if args.local else args.host, args.remote_root) for job in jobs: - process_job(None if args.local else args.host, args.remote_root, job, work_dir, args.movmaker_arg, args.force) + process_job(None if args.local else args.host, args.remote_root, job, work_dir, temp_dir, args.movmaker_arg, args.force) return 0