diff --git a/.gitignore b/.gitignore index 3bb825e..10705f0 100644 --- a/.gitignore +++ b/.gitignore @@ -6,8 +6,10 @@ venv/ # Generated videos / outputs .logs/ +*.log movmaker.log out/ +output/ *.mp4 *.mov *.mkv @@ -19,9 +21,12 @@ input*/ media/ ffmpeg/ node_modules/ -img/ aabenraa/ +# Bundled/static assets: track img/moto_travel.png, ignore generated/downloaded extras +img/.logs/ +img/*.mp3 + # Assistant/project-local context AGENTS.md diff --git a/README.md b/README.md index 2b275c0..4bcfb61 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Flat-folder Python + ffmpeg movie maker, plus optional MVLog worker. - Input: one directory with mixed images/videos/audio + optional `data.txt` - Output: one rendered MP4 with transitions, captions, intro/logo, stamps, and embedded metadata -- Optional worker (`mvlog_worker.py`) syncs/render/publishes MVLog jobs to a remote webserver +- Optional worker (`mvlog_worker.py`) renders/publishes MVLog jobs either locally on the webserver or via the older SSH/SCP remote mode ## Requirements @@ -96,7 +96,7 @@ clip01.mp4 audio: yes ## MVLog worker (`mvlog_worker.py`) -Worker is for MVLog web integration (remote `in-dir` -> local render -> remote `out-dir`). +Worker is for MVLog web integration. Preferred deployment is webserver-local: `/var/www/html/in-dir` -> render on the webserver -> `/var/www/html/out-dir`. Older SSH/SCP remote mode is still available for rollback/manual use. Key points: @@ -107,18 +107,24 @@ Key points: - Publishes output atomically and clears remote cache file - Supports remote mode (SSH/SCP) and `--local` mode -Run once: +Run once on the webserver: + +```bash +./mvlog_worker.py --local --remote-root /var/www/html --work-dir /srv/mvlog/work +``` + +Single job on the webserver: + +```bash +./mvlog_worker.py --local --remote-root /var/www/html --work-dir /srv/mvlog/work --job 20260525_mohnesee +``` + +Older SSH/SCP remote mode from another machine: ```bash ./mvlog_worker.py ``` -Single job: - -```bash -./mvlog_worker.py --job 20260525_mohnesee -``` - Force rerender: ```bash @@ -131,6 +137,25 @@ Pass extra movmaker args: ./mvlog_worker.py --movmaker-arg=--preview ``` +## Webserver deployment + +Recommended deployment keeps this repo checked out on the webserver at: + +```text +/srv/mvlog/app +``` + +The worker writes temporary render data outside git at `/srv/mvlog/work` and publishes public files to `/var/www/html/out-dir`. + +Systemd templates are in `deploy/systemd/`: + +```bash +sudo cp deploy/systemd/mvlog-worker.service /etc/systemd/system/ +sudo cp deploy/systemd/mvlog-worker.timer /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable --now mvlog-worker.timer +``` + ## Notes - This repo does **not** send web push itself; push is handled by MVLog web app server-side. diff --git a/deploy/systemd/mvlog-worker.service b/deploy/systemd/mvlog-worker.service new file mode 100644 index 0000000..fdf5909 --- /dev/null +++ b/deploy/systemd/mvlog-worker.service @@ -0,0 +1,13 @@ +[Unit] +Description=MVLog movmaker worker (webserver local) +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/srv/mvlog/app +ExecStart=/srv/mvlog/app/mvlog_worker.py --local --remote-root /var/www/html --work-dir /srv/mvlog/work +User=hbrain +Group=www-data +UMask=0002 +TimeoutStartSec=12h diff --git a/deploy/systemd/mvlog-worker.timer b/deploy/systemd/mvlog-worker.timer new file mode 100644 index 0000000..78d1284 --- /dev/null +++ b/deploy/systemd/mvlog-worker.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Run MVLog movmaker worker periodically on webserver + +[Timer] +OnBootSec=2min +OnUnitInactiveSec=5min +AccuracySec=30s +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/img/moto_travel.png b/img/moto_travel.png new file mode 100644 index 0000000..60f927f Binary files /dev/null and b/img/moto_travel.png differ diff --git a/mvlog_worker.py b/mvlog_worker.py index ca50f04..40b14d9 100755 --- a/mvlog_worker.py +++ b/mvlog_worker.py @@ -51,8 +51,13 @@ def remote_q(path: str) -> str: return quote(path) -def list_jobs(host: str, remote_root: str) -> list[str]: - cmd = f"cd {remote_q(remote_root)} && find in-dir -mindepth 1 -maxdepth 1 -type d -printf '%f\\n' | sort" +def list_jobs(host: str | None, remote_root: str) -> list[str]: + if host is None: + root = Path(remote_root) / "in-dir" + if not root.is_dir(): + return [] + return sorted(path.name for path in root.iterdir() if path.is_dir() and not path.name.startswith('.')) + cmd = f"cd {remote_q(remote_root)} && find in-dir -mindepth 1 -maxdepth 1 -type d ! -name '.*' -printf '%f\\n' | sort" out = ssh(host, cmd).stdout return [line.strip() for line in out.splitlines() if line.strip()] @@ -80,17 +85,41 @@ def read_article_id(host: str | None, job_dir: str) -> str | None: return None -def acquire_lock(host: str, job_dir: str) -> bool: +def acquire_lock(host: str | None, job_dir: str) -> bool: lock = f"{job_dir}/{LOCK_NAME}" + if host is None: + try: + Path(lock).mkdir() + Path(lock, "started_at").write_text(now() + "\n", encoding="utf-8") + return True + except FileExistsError: + return False cmd = f"mkdir {remote_q(lock)} 2>/dev/null && date -Iseconds > {remote_q(lock + '/started_at')}" return ssh(host, cmd, check=False).returncode == 0 -def release_lock(host: str, job_dir: str) -> None: +def release_lock(host: str | None, job_dir: str) -> None: + if host is None: + shutil.rmtree(Path(job_dir) / LOCK_NAME, ignore_errors=True) + return ssh(host, f"rm -rf {remote_q(job_dir + '/' + LOCK_NAME)}", check=False) -def remote_fingerprint(host: str, job_dir: str) -> str: +def remote_fingerprint(host: str | None, job_dir: str) -> str: + if host is None: + entries = [] + root = Path(job_dir) + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = [d for d in dirnames if d != LOCK_NAME] + for name in filenames: + if name in {STATE_NAME, ENABLED_NAME, PREVIEW_NAME, HIDDEN_NAME, PERMALINK_NAME, ARTICLE_ID_NAME, '.mvlog-hide-map'}: + continue + path = Path(dirpath) / name + rel = path.relative_to(root).as_posix() + st = path.stat() + entries.append([rel, st.st_size, st.st_mtime_ns]) + entries.sort() + return hashlib.sha256(json.dumps(entries, ensure_ascii=False, separators=(",", ":")).encode("utf-8")).hexdigest() script = r''' import hashlib, json, os, sys root = sys.argv[1] @@ -169,15 +198,28 @@ def write_state(host: str | None, job_dir: str, state: dict) -> None: ssh(host, cmd, input_text=data) -def copy_job_from_remote(host: str, job_dir: str, local_input: Path) -> None: +def copy_job_from_remote(host: str | None, job_dir: str, local_input: Path) -> None: local_input.mkdir(parents=True, exist_ok=True) + excludes = {LOCK_NAME, STATE_NAME, PERMALINK_NAME} + if host is None: + src = Path(job_dir) + for path in src.rglob("*"): + if any(part in excludes for part in path.relative_to(src).parts): + continue + dest = local_input / path.relative_to(src) + if path.is_dir(): + dest.mkdir(parents=True, exist_ok=True) + elif path.is_file(): + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(path, dest) + return remote_cmd = ( f"tar --exclude={quote(LOCK_NAME)} --exclude={quote(STATE_NAME)} --exclude={quote(PERMALINK_NAME)} " f"-C {remote_q(job_dir)} -cf - ." ) p1 = subprocess.Popen(["ssh", host, remote_cmd], stdout=subprocess.PIPE) try: - p2 = subprocess.run(["tar", "-xf", "-", "-C", str(local_input)], stdin=p1.stdout, check=True) + subprocess.run(["tar", "-xf", "-", "-C", str(local_input)], stdin=p1.stdout, check=True) assert p1.stdout is not None p1.stdout.close() rc = p1.wait() @@ -190,8 +232,19 @@ def copy_job_from_remote(host: str, job_dir: str, local_input: Path) -> None: -def publish_output(host: str, remote_root: str, local_mp4: Path, old_output: str | None) -> str: +def publish_output(host: str | None, remote_root: str, local_mp4: Path, old_output: str | None) -> str: out_name = local_mp4.name + if host is None: + out_dir = Path(remote_root) / "out-dir" + out_dir.mkdir(parents=True, exist_ok=True) + final = out_dir / out_name + tmp = out_dir / f".{out_name}.tmp" + shutil.copy2(local_mp4, tmp) + tmp.replace(final) + (Path(remote_root) / "cache" / "videos.json").unlink(missing_ok=True) + if old_output and old_output != out_name: + (out_dir / old_output).unlink(missing_ok=True) + return out_name remote_tmp = f"{remote_root}/out-dir/.{out_name}.tmp" remote_final = f"{remote_root}/out-dir/{out_name}" subprocess.run(["scp", str(local_mp4), f"{host}:{remote_tmp}"], check=True) @@ -271,7 +324,7 @@ def render_job(local_input: Path, local_out: Path, extra_args: list[str]) -> Pat return outputs[0] -def process_job(host: str, remote_root: str, job: str, work_dir: Path, extra_args: list[str], force: bool) -> None: +def process_job(host: str | None, remote_root: str, job: str, work_dir: Path, extra_args: list[str], force: bool) -> None: job_dir = f"{remote_root}/in-dir/{job}" article_id = read_article_id(host, job_dir) state = read_state(host, job_dir)