diff --git a/re-encode-publish-Q-Tube.py b/re-encode-publish-Q-Tube.py new file mode 100644 index 0000000..2b8cb62 --- /dev/null +++ b/re-encode-publish-Q-Tube.py @@ -0,0 +1,508 @@ +import os +import json +import base64 +import hashlib +import string +import random +import subprocess +import shlex +import requests +import getpass +import argparse +from datetime import datetime + +# Configuration +API_URL = "http://localhost:12391" +VIDEO_EXTENSIONS = ['.mp4', '.mkv', '.avi', '.mov', '.flv'] +SIZE_LIMIT_MB = 410 +DEFAULT_CODEC = 'h264' +DEFAULT_USE_NVIDIA = False +LOG_FILE = "video-encode-and-publish.log" + +# Logging +def log(msg): + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + line = f"[{timestamp}] {msg}" + print(line) + with open(LOG_FILE, "a") as log_file: + log_file.write(line + "\n") + +# Utility functions +def slugify(text, max_length=32): + import re + text = text.lower() + text = re.sub(r'[^a-z0-9]+', '-', text).strip('-') + return text[:max_length] + +def generate_short_id(length=6): + alphabet = string.ascii_letters + string.digits + return ''.join(random.choices(alphabet, k=length)) + +def get_video_duration_and_size(filepath): + result = subprocess.run([ + 'ffprobe', '-v', 'error', '-show_entries', + 'format=duration,size', '-of', 'json', filepath + ], capture_output=True, text=True) + metadata = json.loads(result.stdout) + duration = float(metadata['format']['duration']) + size = int(metadata['format']['size']) + return duration, size + +def generate_thumbnail(video_path, timestamp, output_suffix): + thumbnail_path_webp = f"{video_path}.{output_suffix}.webp" + thumbnail_path_jpeg = f"{video_path}.{output_suffix}.jpg" + + # Attempt WebP first + try: + subprocess.run([ + 'ffmpeg', '-ss', str(timestamp), '-i', video_path, + '-vframes', '1', '-vf', 'scale=320:-1', '-y', thumbnail_path_webp + ], check=True) + if os.path.exists(thumbnail_path_webp): + return thumbnail_path_webp + except subprocess.CalledProcessError as e: + log(f"[WARNING] WebP encoding returned error: {e}. Checking if file was written anyway...") + + # Accept WebP if it was written despite error + if os.path.exists(thumbnail_path_webp): + log(f"[RECOVERED] WebP thumbnail exists despite ffmpeg error.") + return thumbnail_path_webp + + # Fallback to JPEG + try: + subprocess.run([ + 'ffmpeg', '-ss', str(timestamp), '-i', video_path, + '-vframes', '1', '-vf', 'scale=320:-1', '-c:v', 'mjpeg', '-y', thumbnail_path_jpeg + ], check=True) + if os.path.exists(thumbnail_path_jpeg): + log(f"[FALLBACK] JPEG thumbnail created: {thumbnail_path_jpeg}") + return thumbnail_path_jpeg + except subprocess.CalledProcessError as e: + log(f"[ERROR] JPEG thumbnail also failed at {timestamp}s: {e}") + + log(f"[SKIPPED] No thumbnail generated for timestamp {timestamp}s.") + return None + + + +def generate_all_thumbnails(video_path, duration): + thumbnails = [] + intervals = [duration * i / 4 for i in range(1, 5)] + for idx, ts in enumerate(intervals, start=1): + thumb_path = generate_thumbnail(video_path, ts, f"thumb{idx}") + if thumb_path: + thumbnails.append(f"data:image/webp;base64,{base64_encode_file(thumb_path)}") + return thumbnails + +def base64_encode_file(path): + with open(path, 'rb') as f: + return base64.b64encode(f.read()).decode('utf-8') + +def get_private_key(): + return getpass.getpass("Enter your QDN private key: ") + +def post(endpoint, data): + response = requests.post(f"{API_URL}/{endpoint}", json=data) + response.raise_for_status() + return response.json() + +def get_api_key(): + key_paths = [ + os.path.expanduser("~/.qortal/apikey.txt"), + os.path.expanduser("~/qortal/apikey.txt") + ] + for path in key_paths: + if os.path.exists(path): + with open(path, 'r') as f: + return f.read().strip() + return input("Enter your Qortal API key: ").strip() + + +def build_sign_publish_from_file(service, identifier, name, file_path, private_key, dry_run=False, metadata={}): + """ + Publishes a file to QDN with metadata and signs it locally. + + Args: + service: 'VIDEO' or 'DOCUMENT' + identifier: QDN identifier + name: QDN channel name + file_path: Path to file + private_key: Base58 private key + dry_run: If True, log actions but do not execute + metadata: Dictionary containing optional title, description, category, fee, preview + """ + api_key = get_api_key() + + if not os.path.exists(file_path): + raise FileNotFoundError(f"File not found: {file_path}") + + # Set defaults if metadata keys are missing + params = {} + + if "title" in metadata: + params["title"] = metadata["title"] + + if "fullDescription" in metadata: + params["description"] = metadata["fullDescription"] + + category_val = metadata.get("category") + if isinstance(category_val, str) and category_val.strip(): + params["category"] = category_val.strip() + + params["fee"] = metadata.get("fee", "01000000") + params["preview"] = str(metadata.get("preview", "false")).lower() + + url = f"{API_URL}/arbitrary/{service}/{name}/{identifier}" + + if dry_run: + log(f"[DRY RUN] Would publish {file_path} to {url} with metadata: {params}") + return + + # Step 1: Build raw transaction + headers = { + 'X-API-KEY': api_key, + 'Content-Type': 'text/plain', + 'Accept': 'text/plain' + } + + log(f"[PUBLISH TX] Creating raw transaction for {identifier}") + log(f"[API URL] {url}") + log(f"[HEADERS] {headers}") + log(f"[PARAMS] {params}") + log(f"[BODY] {file_path}") + + try: + response = requests.post( + url, + headers=headers, + params=params, + data=file_path.encode() + ) + except Exception as e: + log(f"[EXCEPTION] During POST to /arbitrary: {str(e)}") + raise + + log(f"[RESPONSE CODE] {response.status_code}") + log(f"[RESPONSE TEXT] {response.text}") + + raw_tx = response.text.strip() + + # Step 2: Sign the transaction + sign_response = requests.post( + f"{API_URL}/transactions/sign", + headers={ + 'Content-Type': 'application/json', + 'Accept': 'application/json' + }, + json={ + "privateKey": private_key, + "transactionBytes": raw_tx + } + ) + + if sign_response.status_code != 200: + log(f"[ERROR] Transaction signing failed: {sign_response.status_code} {sign_response.text}") + raise Exception("Signing transaction failed") + + signed_tx = sign_response.text.strip() + log(f"[SIGNED TX] {signed_tx[:80]}...") + + # Step 3: Broadcast the signed transaction + broadcast_response = requests.post( + f"{API_URL}/transactions/process", + headers={ + 'Content-Type': 'text/plain', + 'Accept': 'text/plain' + }, + data=signed_tx + ) + + if broadcast_response.status_code != 200: + log(f"[ERROR] Broadcast failed: {broadcast_response.status_code} {broadcast_response.text}") + raise Exception("Broadcast failed") + + log(f"[PUBLISHED] {identifier} ({service})") + + +def encoder_available(encoder): + result = subprocess.run(['ffmpeg', '-hide_banner', '-encoders'], capture_output=True, text=True) + return encoder in result.stdout + +def should_reencode(video_path): + try: + # Get codec and size + result = subprocess.run([ + 'ffprobe', '-v', 'error', '-select_streams', 'v:0', + '-show_entries', 'stream=codec_name', '-of', + 'default=noprint_wrappers=1:nokey=1', video_path + ], capture_output=True, text=True) + + codec = result.stdout.strip() + _, size_bytes = get_video_duration_and_size(video_path) + size_mb = size_bytes / (1024 * 1024) + + unsupported_codecs = ['hevc', 'vp9', 'av1'] # Common ones Chromium won't hardware-decode + if codec in unsupported_codecs: + log(f"[CHECK] Re-encoding required due to unsupported codec: {codec}") + return True + if size_mb > SIZE_LIMIT_MB: + log(f"[CHECK] Re-encoding required due to file size: {round(size_mb, 2)}MB") + return True + + log(f"[CHECK] Skipping re-encoding (codec: {codec}, size: {round(size_mb, 2)}MB)") + return False + + except Exception as e: + log(f"[ERROR] Failed to check codec or size: {e}") + return True # Fail safe: re-encode if unsure + + +def reencode_video(video_path, codec=DEFAULT_CODEC, use_nvidia=DEFAULT_USE_NVIDIA): + base_name, _ = os.path.splitext(video_path) + output_path = f"{base_name}_{codec}_encoded.mp4" + + duration, _ = get_video_duration_and_size(video_path) + bitrate_override = template.get("bitrate") if 'template' in locals() else None + + try: + if codec == 'av1': + encoder = 'libsvtav1' + crf = "30" + if not encoder_available(encoder): + log(f"[ERROR] Encoder '{encoder}' not available. Falling back to 'libx264'.") + encoder = 'libx264' + codec = 'h264' + use_nvidia = False + elif codec == 'h264' and use_nvidia: + encoder = 'h264_nvenc' + else: + encoder = 'libx264' + + if not encoder_available(encoder): + log(f"[ERROR] Encoder '{encoder}' not available. Falling back to 'libx264'.") + encoder = 'libx264' + + # Build ffmpeg command + command = ['ffmpeg', '-hwaccel', 'auto', '-i', video_path] + + if codec == 'av1': + command += ['-c:v', encoder, '-crf', crf] + else: + if bitrate_override: + target_bitrate = bitrate_override + else: + if duration <= 90: + target_bitrate = "500k" + elif duration <= 300: + target_bitrate = "500k" + else: + target_bitrate = "400k" + command += ['-c:v', encoder, '-b:v', target_bitrate] + + command += ['-c:a', 'libvorbis', '-y', output_path] + + log(f"[ENCODING] Running ffmpeg: {' '.join(command)}") + subprocess.run(command, check=True) + except subprocess.CalledProcessError: + log(f"[ERROR] Encoding failed with {encoder}.") + return None + + # Move original to ORIGINALS + originals_dir = os.path.join(os.path.dirname(video_path), 'ORIGINALS') + os.makedirs(originals_dir, exist_ok=True) + os.rename(video_path, os.path.join(originals_dir, os.path.basename(video_path))) + + _, final_size = get_video_duration_and_size(output_path) + if final_size > SIZE_LIMIT_MB * 1024 * 1024: + too_large_dir = os.path.join(os.path.dirname(video_path), 'too_large') + os.makedirs(too_large_dir, exist_ok=True) + os.rename(output_path, os.path.join(too_large_dir, os.path.basename(output_path))) + log(f"[TOO LARGE] {os.path.basename(output_path)} moved to 'too_large' folder") + return None + + log(f"[ENCODED] {output_path} ({final_size // 1024 // 1024}MB)") + return output_path + + +def load_metadata(path): + if os.path.exists(path): + with open(path) as f: + return json.load(f) + log(f"[INFO] No QDN.json found. Using defaults.") + return { + "codec": DEFAULT_CODEC, + "use_nvidia": DEFAULT_USE_NVIDIA + } + +def save_metadata_template(path, metadata): + with open(path, 'w') as f: + json.dump(metadata, f, indent=2) + log(f"Saved QDN.json template to: {path}") + +def prompt_for_metadata(): + title = input("Title: ") + description = input("Full Description: ") + html_description = input("HTML Description: ") + category = int(input("Category (number): ")) + codec = input("Video codec (av1/h264): ").strip().lower() or DEFAULT_CODEC + if codec not in ['av1', 'h264']: + codec = DEFAULT_CODEC + use_nvidia = input("Use NVIDIA hardware acceleration? (yes/no): ").strip().lower() == 'yes' + return title, description, html_description, category, codec, use_nvidia + +def publish_qtube(video_path, private_key, mode='auto', dry_run=False): + if not video_path: + return + + base_name = os.path.splitext(os.path.basename(video_path))[0] + root_dir = os.path.dirname(video_path) + metadata_path = os.path.join(root_dir, 'QDN.json') + template = load_metadata(metadata_path) if mode == 'auto' else {} + + title = template.get("title") + fullDescription = template.get("fullDescription") + htmlDescription = template.get("htmlDescription") + category = template.get("category", 9) + name = template.get("videoReference", {}).get("name") or input("Enter channel name: ") + codec = template.get("codec", DEFAULT_CODEC) + use_nvidia = template.get("use_nvidia", DEFAULT_USE_NVIDIA) + + if mode == 'manual' or not all([title, fullDescription, htmlDescription]): + use_auto = input("[PROMPT] QDN.json loaded, would you like to use it as-is? (yes/no): ").strip().lower() + if use_auto != 'yes': + title, fullDescription, htmlDescription, category, codec, use_nvidia = prompt_for_metadata() + + slug = slugify(title) + base_slug = f"qtube_vid_{slug}" + identifier = None + short_id = None + + try: + response = requests.get( + f"{API_URL}/arbitrary/resources/searchsimple", + params={ + "service": "DOCUMENT", + "name": name, + "limit": 0, + "reverse": "true" + } + ) + if response.status_code == 200: + results = response.json() + for item in results: + ident = item.get("identifier", "") + if ident.startswith(base_slug) and ident.endswith("_metadata"): + identifier = ident.replace("_metadata", "") + short_id = identifier.split("_")[-1] + log(f"[REUSE] Existing publish found. Reusing identifier: {identifier}") + break + except Exception as e: + log(f"[WARNING] Could not query searchSimple: {e}") + + # If not found, create new identifier + if not identifier: + short_id = generate_short_id() + identifier = f"{base_slug}_{short_id}" + log(f"[NEW] No existing identifier found. Creating new: {identifier}") + comments_id = f"qtube_vid__cm_{short_id}" + metadata_identifier = f"{identifier}_metadata" + + duration, file_size = get_video_duration_and_size(video_path) + video_image_path = generate_thumbnail(video_path, 1, "poster") + extracts = generate_all_thumbnails(video_path, duration) + + metadata = { + "title": title, + "version": 1, + "fullDescription": fullDescription, + "htmlDescription": htmlDescription, + "videoImage": f"data:image/webp;base64,{base64_encode_file(video_image_path)}" if video_image_path else "", + "videoReference": { + "name": name, + "identifier": identifier, + "service": "VIDEO" + }, + "extracts": extracts, + "commentsId": comments_id, + "category": category, + "subcategory": template.get("subcategory", ""), + "code": template.get("code", short_id), + "videoType": "video/mp4", + "filename": os.path.basename(video_path), + "fileSize": file_size, + "duration": duration, + "codec": codec, + "use_nvidia": use_nvidia + } + + if mode == 'manual': + save_metadata_template(metadata_path, metadata) + + # Save metadata JSON file for publishing + metadata_file_path = os.path.join(root_dir, f"{identifier}_metadata.json") + with open(metadata_file_path, 'w') as f: + json.dump(metadata, f, indent=2) + log(f"[WRITE] Saved metadata JSON to: {metadata_file_path}") + + # Publish video + log(f"Publishing VIDEO: {identifier}") + build_sign_publish_from_file("VIDEO", identifier, name, video_path, private_key, dry_run) + + # Publish metadata + log(f"Publishing METADATA: {metadata_identifier}") + build_sign_publish_from_file("DOCUMENT", metadata_identifier, name, metadata_file_path, private_key, dry_run) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--mode', choices=['auto', 'manual'], default='auto') + parser.add_argument('--dry-run', action='store_true') + args = parser.parse_args() + + private_key = get_private_key() + + for root, dirs, files in os.walk(os.getcwd()): + # Skip folders we don't want to process + if 'ORIGINALS' in dirs: + dirs.remove('ORIGINALS') + if 'too_large' in dirs: + dirs.remove('too_large') + + qdn_template_cache = None + apply_template_for_folder = False + + for file in files: + if any(file.lower().endswith(ext) for ext in VIDEO_EXTENSIONS): + try: + full_path = os.path.join(root, file) + + # Per-folder QDN.json check + if not qdn_template_cache: + metadata_path = os.path.join(root, 'QDN.json') + if os.path.exists(metadata_path): + qdn_template_cache = load_metadata(metadata_path) + print(f"Found QDN.json in: {root}") + use_for_folder = input("Use this QDN.json for all videos in this folder? (yes/no): ").strip().lower() + apply_template_for_folder = (use_for_folder == 'yes') + + template = qdn_template_cache if apply_template_for_folder else {} + + codec = template.get("codec", DEFAULT_CODEC) + use_nvidia = template.get("use_nvidia", DEFAULT_USE_NVIDIA) + + if should_reencode(full_path): + reencoded_path = reencode_video(full_path, codec=codec, use_nvidia=use_nvidia) + if reencoded_path: + publish_qtube(reencoded_path, private_key, mode=args.mode, dry_run=args.dry_run) + else: + publish_qtube(full_path, private_key, mode=args.mode, dry_run=args.dry_run) + + except Exception as e: + log(f"Failed to process {file}: {e}") + + + + +if __name__ == "__main__": + main() +