From 0e90b453246198dcbdbfdaaf23ebc3553e05539a Mon Sep 17 00:00:00 2001 From: calesthio Date: Sat, 4 Apr 2026 13:08:54 -0700 Subject: [PATCH] Remove QA tests that make paid API calls test_01_tts.py (ElevenLabs), test_02_image_gen.py (DALL-E + FLUX), and test_03_music.py (ElevenLabs Music) were burning ~$0.44 per run against live API keys. These were run repeatedly and exhausted the ElevenLabs starter plan quota (30K characters). --- tests/qa/test_01_tts.py | 80 ------------------------------- tests/qa/test_02_image_gen.py | 88 ----------------------------------- tests/qa/test_03_music.py | 62 ------------------------ 3 files changed, 230 deletions(-) delete mode 100644 tests/qa/test_01_tts.py delete mode 100644 tests/qa/test_02_image_gen.py delete mode 100644 tests/qa/test_03_music.py diff --git a/tests/qa/test_01_tts.py b/tests/qa/test_01_tts.py deleted file mode 100644 index 524dd3c7..00000000 --- a/tests/qa/test_01_tts.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python3 -"""QA Test 01: TTS voice generation via ElevenLabs.""" - -import sys, os, json, time -from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) - -from lib.env_loader import load_env -load_env() - -from tools.audio.elevenlabs_tts import ElevenLabsTTS - -OUT = os.path.join(os.path.dirname(__file__), "output") -os.makedirs(OUT, exist_ok=True) - -tool = ElevenLabsTTS() -print(f"Tool status: {tool.get_status()}") - -# Test 1: Short narration -print("\n--- Test 1: Short narration ---") -r1 = tool.execute({ - "text": "Welcome to OpenMontage. Let's build something amazing together.", - "output_path": os.path.join(OUT, "tts_short.mp3"), -}) -print(f"Success: {r1.success}, Cost: ${r1.cost_usd:.4f}, Duration: {r1.duration_seconds:.2f}s") -if r1.error: print(f"Error: {r1.error}") -if r1.artifacts: print(f"Artifacts: {r1.artifacts}") - -# Test 2: Longer paragraph with technical content -print("\n--- Test 2: Technical narration ---") -r2 = tool.execute({ - "text": ( - "Quantum computing leverages quantum mechanical phenomena like superposition and entanglement " - "to process information in fundamentally different ways than classical computers. " - "While a classical bit can be either zero or one, a quantum bit, or qubit, can exist in " - "a superposition of both states simultaneously. This allows quantum computers to explore " - "many possible solutions at once, making them exceptionally powerful for certain types of problems." - ), - "output_path": os.path.join(OUT, "tts_technical.mp3"), -}) -print(f"Success: {r2.success}, Cost: ${r2.cost_usd:.4f}, Duration: {r2.duration_seconds:.2f}s") -if r2.error: print(f"Error: {r2.error}") -if r2.artifacts: print(f"Artifacts: {r2.artifacts}") - -# Test 3: Emotional / storytelling narration -print("\n--- Test 3: Storytelling narration ---") -r3 = tool.execute({ - "text": ( - "Picture this. You wake up one morning, check your phone, and discover that overnight, " - "your side project went viral. Thousands of people are using it. Messages are flooding in. " - "This isn't a dream. This is what happens when you build something people actually need." - ), - "output_path": os.path.join(OUT, "tts_story.mp3"), -}) -print(f"Success: {r3.success}, Cost: ${r3.cost_usd:.4f}, Duration: {r3.duration_seconds:.2f}s") -if r3.error: print(f"Error: {r3.error}") -if r3.artifacts: print(f"Artifacts: {r3.artifacts}") - -# Probe outputs with ffprobe -import subprocess -for name in ["tts_short.mp3", "tts_technical.mp3", "tts_story.mp3"]: - path = os.path.join(OUT, name) - if os.path.exists(path): - probe = subprocess.run( - ["ffprobe", "-v", "quiet", "-print_format", "json", "-show_format", "-show_streams", path], - capture_output=True, text=True - ) - info = json.loads(probe.stdout) - fmt = info.get("format", {}) - streams = info.get("streams", [{}]) - audio = streams[0] if streams else {} - print(f"\n[{name}] Duration: {fmt.get('duration', '?')}s, " - f"Sample rate: {audio.get('sample_rate', '?')}Hz, " - f"Channels: {audio.get('channels', '?')}, " - f"Codec: {audio.get('codec_name', '?')}, " - f"Size: {os.path.getsize(path)} bytes") - else: - print(f"\n[{name}] FILE NOT FOUND") - -print("\n=== TTS TEST COMPLETE ===") diff --git a/tests/qa/test_02_image_gen.py b/tests/qa/test_02_image_gen.py deleted file mode 100644 index 3718598c..00000000 --- a/tests/qa/test_02_image_gen.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python3 -"""QA Test 02: Image generation via OpenAI (DALL-E 3) and fal.ai (FLUX).""" - -import sys, os, json, time -from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) - -from lib.env_loader import load_env -load_env() - -from tools.graphics.image_gen import ImageGen - -OUT = os.path.join(os.path.dirname(__file__), "output") -os.makedirs(OUT, exist_ok=True) - -tool = ImageGen() -print(f"Tool status: {tool.get_status()}") - -# Test 1: DALL-E — professional diagram -print("\n--- Test 1: DALL-E professional diagram ---") -r1 = tool.execute({ - "prompt": "A clean, professional infographic showing the 5 stages of a video production pipeline: Idea, Script, Assets, Edit, Publish. Flat design, blue and amber color scheme, white background, no text.", - "width": 1280, - "height": 720, - "provider": "openai", - "output_path": os.path.join(OUT, "img_dalle_diagram.png"), -}) -print(f"Success: {r1.success}, Cost: ${r1.cost_usd:.4f}") -if r1.error: print(f"Error: {r1.error}") -if r1.artifacts: print(f"Artifacts: {r1.artifacts}") - -# Test 2: DALL-E — cinematic scene -print("\n--- Test 2: DALL-E cinematic scene ---") -r2 = tool.execute({ - "prompt": "A futuristic control room with holographic displays showing data visualizations, cinematic lighting, wide angle, film grain, warm tones", - "width": 1280, - "height": 720, - "provider": "openai", - "output_path": os.path.join(OUT, "img_dalle_cinematic.png"), -}) -print(f"Success: {r2.success}, Cost: ${r2.cost_usd:.4f}") -if r2.error: print(f"Error: {r2.error}") -if r2.artifacts: print(f"Artifacts: {r2.artifacts}") - -# Test 3: FLUX via fal.ai — abstract tech -print("\n--- Test 3: FLUX abstract tech ---") -r3 = tool.execute({ - "prompt": "Abstract visualization of neural network connections, glowing nodes and edges, dark background, neon blue and purple, high detail, 8k render", - "width": 1280, - "height": 720, - "provider": "flux", - "output_path": os.path.join(OUT, "img_flux_abstract.png"), -}) -print(f"Success: {r3.success}, Cost: ${r3.cost_usd:.4f}") -if r3.error: print(f"Error: {r3.error}") -if r3.artifacts: print(f"Artifacts: {r3.artifacts}") - -# Test 4: FLUX — character/mascot -print("\n--- Test 4: FLUX character illustration ---") -r4 = tool.execute({ - "prompt": "Friendly robot mascot character, simple geometric design, holding a film clapboard, isometric view, clean white background, flat illustration style", - "width": 1024, - "height": 1024, - "provider": "flux", - "output_path": os.path.join(OUT, "img_flux_mascot.png"), -}) -print(f"Success: {r4.success}, Cost: ${r4.cost_usd:.4f}") -if r4.error: print(f"Error: {r4.error}") -if r4.artifacts: print(f"Artifacts: {r4.artifacts}") - -# Probe outputs -import subprocess -for name in ["img_dalle_diagram.png", "img_dalle_cinematic.png", "img_flux_abstract.png", "img_flux_mascot.png"]: - path = os.path.join(OUT, name) - if os.path.exists(path): - probe = subprocess.run( - ["ffprobe", "-v", "quiet", "-print_format", "json", "-show_streams", path], - capture_output=True, text=True - ) - info = json.loads(probe.stdout) - stream = info.get("streams", [{}])[0] - print(f"\n[{name}] {stream.get('width', '?')}x{stream.get('height', '?')}, " - f"Format: {stream.get('codec_name', '?')}, " - f"Size: {os.path.getsize(path)} bytes") - else: - print(f"\n[{name}] FILE NOT FOUND") - -print("\n=== IMAGE GEN TEST COMPLETE ===") diff --git a/tests/qa/test_03_music.py b/tests/qa/test_03_music.py deleted file mode 100644 index a8490b67..00000000 --- a/tests/qa/test_03_music.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python3 -"""QA Test 03: Music generation via ElevenLabs.""" - -import sys, os, json -from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) - -from lib.env_loader import load_env -load_env() - -from tools.audio.music_gen import MusicGen - -OUT = os.path.join(os.path.dirname(__file__), "output") -os.makedirs(OUT, exist_ok=True) - -tool = MusicGen() -print(f"Tool status: {tool.get_status()}") - -# Test 1: Upbeat tech background -print("\n--- Test 1: Upbeat tech background ---") -r1 = tool.execute({ - "prompt": "Upbeat electronic background music, 120 BPM, energetic but not overwhelming, suitable for a tech explainer video", - "duration_seconds": 30, - "output_path": os.path.join(OUT, "music_upbeat.mp3"), -}) -print(f"Success: {r1.success}, Cost: ${r1.cost_usd:.4f}") -if r1.error: print(f"Error: {r1.error}") -if r1.artifacts: print(f"Artifacts: {r1.artifacts}") - -# Test 2: Calm ambient -print("\n--- Test 2: Calm ambient ---") -r2 = tool.execute({ - "prompt": "Calm ambient background music, soft piano and strings, 80 BPM, reflective mood, suitable for documentary narration", - "duration_seconds": 30, - "output_path": os.path.join(OUT, "music_calm.mp3"), -}) -print(f"Success: {r2.success}, Cost: ${r2.cost_usd:.4f}") -if r2.error: print(f"Error: {r2.error}") -if r2.artifacts: print(f"Artifacts: {r2.artifacts}") - -# Probe outputs -import subprocess -for name in ["music_upbeat.mp3", "music_calm.mp3"]: - path = os.path.join(OUT, name) - if os.path.exists(path): - probe = subprocess.run( - ["ffprobe", "-v", "quiet", "-print_format", "json", "-show_format", "-show_streams", path], - capture_output=True, text=True - ) - info = json.loads(probe.stdout) - fmt = info.get("format", {}) - streams = info.get("streams", [{}]) - audio = streams[0] if streams else {} - print(f"\n[{name}] Duration: {fmt.get('duration', '?')}s, " - f"Sample rate: {audio.get('sample_rate', '?')}Hz, " - f"Channels: {audio.get('channels', '?')}, " - f"Codec: {audio.get('codec_name', '?')}, " - f"Size: {os.path.getsize(path)} bytes") - else: - print(f"\n[{name}] FILE NOT FOUND") - -print("\n=== MUSIC GEN TEST COMPLETE ===")