/** * PhotoIntelDebugScreen — Debug tool for testing MiMo photo intelligence quality. * * Takes a photo (camera or gallery), sends it to MiMo with BOTH the current prompt * and an enhanced prompt (with GPS, compass, time context), and shows raw responses * side-by-side so Bekky can judge data extraction quality. * * Privacy: GPS is coarse (rounded to ~1 square mile). No precise location stored. */ import React, { useState } from 'react'; import { ActivityIndicator, Alert, Image, Platform, Pressable, ScrollView, StyleSheet, Text, TextInput, View, } from 'react-native'; import * as ImagePicker from 'expo-image-picker'; import * as FileSystem from 'expo-file-system'; import { manipulateAsync, SaveFormat } from 'expo-image-manipulator'; import { SafeAreaView } from 'react-native-safe-area-context'; // ── Config ── const proxyEndpoint = 'https://mimo-api.129-212-237-189.sslip.io/v1/chat/completions'; const xiaomiModel = 'mimo-v2-omni'; const maxImageDimension = 512; const jpegQuality = 0.4; const requestTimeoutMs = 30000; // ── Colors (match app) ── const deepLeaf = '#2E6B3F'; const freshSprout = '#78C66A'; const softSage = '#BFD9B4'; const warmCream = '#F8F4E8'; const parchment = '#FDF4DB'; // ── Types ── type PhotoType = 'environment' | 'placement' | 'setup' | 'progress'; interface TestResult { photoType: PhotoType; promptLabel: string; prompt: string; rawResponse: string; parsedSummary: string; duration: number; success: boolean; error?: string; } interface ContextData { compassHeading: string; // N, NE, E, SE, S, SW, W, NW coarseLat: string; coarseLon: string; timeOfDay: string; indoorOutdoor: 'indoor' | 'outdoor' | ''; notes: string; } // ── Coarse GPS rounding (~1 mile / ~1.6km resolution) ── function coarseRound(value: number): number { // Round to 2 decimal places ≈ 1.1km resolution at equator return Math.round(value * 100) / 100; } // ── Compass helpers ── const COMPASS_DIRECTIONS = ['N', 'NE', 'E', 'SE', 'S', 'SW', 'W', 'NW'] as const; function compassToDescription(dir: string): string { const map: Record = { 'N': 'facing north', 'NE': 'facing northeast', 'E': 'facing east', 'SE': 'facing southeast', 'S': 'facing south', 'SW': 'facing southwest', 'W': 'facing west', 'NW': 'facing northwest', }; return map[dir] || 'unknown direction'; } function sunDirectionHint(dir: string, hour: number): string { // Rough sun position by compass direction and time // Northern hemisphere defaults (Da Lat, Vietnam ~12°N) if (hour >= 6 && hour < 10) return 'morning sun coming from the east'; if (hour >= 10 && hour < 14) return 'midday sun from nearly overhead (slightly south at this latitude)'; if (hour >= 14 && hour < 18) return 'afternoon sun coming from the west'; return 'sun is low or below horizon'; } // ── Prompts ── function buildCurrentPrompt(photoType: PhotoType, plantName?: string): string { const promptByType: Record = { environment: 'You are Pixie, a plant care companion. Analyze this environment photo and describe what you observe about the growing conditions. Focus on: light exposure (direction, intensity, hours), airflow, protection from elements, indoor vs outdoor setting, nearby structures. Return ONLY a JSON object: { "summary": "2-3 sentence natural description of the environment" }', placement: 'You are Pixie, a plant care companion. Analyze this placement photo and describe where exactly this plant sits within its environment. Focus on: proximity to windows or light sources, whether it receives direct or indirect light, nearby plants or objects, shelter from drafts or heat sources, height level (floor, shelf, hanging). Return ONLY a JSON object: { "summary": "2-3 sentence natural description of the plant\'s specific placement location" }', setup: 'You are Pixie, a plant care companion. Analyze this setup photo and describe what you observe about the plant\'s growing setup. Focus on: pot type and size, soil/medium visible, drainage, any grow lights, support structures, arrangement. Return ONLY a JSON object: { "summary": "2-3 sentence natural description of the setup" }', progress: 'You are Pixie, a plant care companion. Analyze this progress photo and describe what you observe about the plant\'s current state. Focus on: overall health, growth vigor, leaf color and condition, canopy density, any visible changes. Return ONLY a JSON object: { "summary": "2-3 sentence natural description of growth and health" }', }; let prompt = promptByType[photoType]; if (plantName) { prompt += `\n\nThis is a ${plantName}.`; } return prompt; } function buildEnhancedPrompt( photoType: PhotoType, ctx: ContextData, plantName?: string, ): string { const now = new Date(); const hour = now.getHours(); const timeStr = now.toLocaleTimeString('en-US', { hour: '2-digit', minute: '2-digit' }); const dateStr = now.toLocaleDateString('en-US', { weekday: 'long', month: 'short', day: 'numeric' }); const promptByType: Record = { environment: `You are Pixie, a plant care companion analyzing an ENVIRONMENT photo. Describe the growing conditions you observe. Be specific and structured. Focus on these aspects — if you can determine them from the image: - **Light**: direction, intensity (bright/medium/low/direct/indirect/dappled/shade), estimated hours of exposure - **Exposure type**: full sun, partial sun, morning sun only, afternoon sun only, shade, deep shade - **Setting**: indoor vs outdoor vs covered outdoor (patio/porch/greenhouse) - **Airflow**: open air, enclosed, sheltered, windy indicators - **Protection**: from rain, wind, frost, extreme heat - **Nearby structures**: walls, fences, trees, buildings that affect light/wind - **Ground surface**: soil, concrete, grass, mulch, gravel, decking - **Plant density**: isolated plant, garden bed, crowded, jungle-like ${ctx.indoorOutdoor ? `User states this is an ${ctx.indoorOutdoor} setting.` : ''} ${ctx.notes ? `User notes: "${ctx.notes}"` : ''} Return a JSON object: { "summary": "2-3 sentence overview", "light_direction": "east|west|south|north|overhead|unclear", "light_intensity": "bright_direct|bright_indirect|medium|low|deep_shade|unclear", "exposure_type": "full_sun|partial_sun|morning_only|afternoon_only|shade|unclear", "setting": "indoor|outdoor|covered_outdoor|unclear", "airflow": "open|sheltered|enclosed|unclear", "protection_level": "exposed|partial_shelter|well_sheltered|unclear", "plant_density": "isolated|garden_bed|crowded|jungle_like|unclear", "confidence": "high|medium|low", "uncertainty_notes": "any doubts about what you see" }`, placement: `You are Pixie, a plant care companion analyzing a PLACEMENT photo. Describe where exactly this plant sits within its environment. Be specific. Focus on: - **Light source proximity**: how close to windows, openings, or direct sky - **Light type received**: direct sun, bright indirect, medium indirect, low, artificial - **Height level**: ground, low shelf, table, high shelf, hanging, mounted - **Nearby objects**: other plants, walls, furniture, heat/cool sources - **Microclimate**: sheltered corner, open exposure, drafty, humid pocket - **Orientation**: which direction the plant faces (if discernible) ${ctx.compassHeading ? `The photographer is ${compassToDescription(ctx.compassHeading)}. ${sunDirectionHint(ctx.compassHeading, hour)}` : ''} ${ctx.indoorOutdoor ? `Setting: ${ctx.indoorOutdoor}.` : ''} ${ctx.notes ? `User notes: "${ctx.notes}"` : ''} Return a JSON object: { "summary": "2-3 sentence overview", "light_source": "direct_window|indirect_window|skylight|grow_light|open_sky|unclear", "light_type": "direct_sun|bright_indirect|medium_indirect|low|artificial|unclear", "height_level": "ground|low|table|high|hanging|mounted|unclear", "nearby_heat_cold_sources": "none|heater|AC|oven|radiator|unclear", "microclimate": "open|sheltered_corner|drafty|humid_pocket|unclear", "facing_direction": "north|south|east|west|unclear", "confidence": "high|medium|low", "uncertainty_notes": "any doubts" }`, setup: `You are Pixie, a plant care companion analyzing a SETUP photo. Describe the plant's growing setup. Be specific. Focus on: - **Container**: pot type (terracotta, plastic, ceramic, fabric, ground), size, color - **Soil/medium**: visible type (potting mix, garden soil, sandy, mulch, moss, hydroponic) - **Drainage**: drainage holes visible, saucer, self-watering, raised bed - **Support**: trellis, stake, cage, moss pole, hanging, none - **Artificial light**: grow light type, distance, color temperature - **Arrangement**: single pot, grouped, raised bed, in-ground, window box - **Mulch/topping**: visible mulch, pebbles, moss, bare soil ${ctx.notes ? `User notes: "${ctx.notes}"` : ''} Return a JSON object: { "summary": "2-3 sentence overview", "container_type": "terracotta|plastic|ceramic|fabric|ground|raised_bed|other|unclear", "container_size": "small|medium|large|very_large|unclear", "soil_type": "potting_mix|garden_soil|sandy|mulch|moss|hydroponic|unclear", "drainage": "good|moderate|poor|unclear", "support": "none|trellis|stake|cage|moss_pole|hanging|unclear", "has_grow_light": true|false|null, "mulch_topping": "none|mulch|pebbles|moss|other|unclear", "confidence": "high|medium|low", "uncertainty_notes": "any doubts" }`, progress: `You are Pixie, a plant care companion analyzing a PROGRESS photo. Describe the plant's current state. Be specific about what you see. Focus on: - **Overall health**: thriving, healthy, stressed, declining, critical - **Growth vigor**: rapid, steady, slow, stalled, dormant - **Leaf condition**: color (dark green, light green, yellow, brown, variegated), texture, size - **Canopy density**: full, moderate, sparse, leggy, compact - **Visible issues**: yellow leaves, brown tips, spots, pests, wilting, etiolation - **Positive signs**: new growth, flowers, fruit, root growth, sturdy stems - **Changes since last**: if any context provided about previous state ${ctx.notes ? `User notes: "${ctx.notes}"` : ''} Return a JSON object: { "summary": "2-3 sentence overview", "overall_health": "thriving|healthy|stressed|declining|critical|unclear", "growth_vigor": "rapid|steady|slow|stalled|dormant|unclear", "leaf_color": "dark_green|light_green|yellowing|browning|variegated|mixed|unclear", "canopy_density": "full|moderate|sparse|leggy|compact|unclear", "visible_issues": ["list", "of", "issues"] or [], "positive_signs": ["list", "of", "signs"] or [], "confidence": "high|medium|low", "uncertainty_notes": "any doubts" }`, }; let prompt = promptByType[photoType]; // Add spatial context const contextLines: string[] = []; if (ctx.coarseLat && ctx.coarseLon) { contextLines.push(`Approximate location: ${ctx.coarseLat}°, ${ctx.coarseLon}° (coarse, ~1 mile radius). Use this for climate/season estimation.`); } if (ctx.timeOfDay || timeStr) { contextLines.push(`Photo taken at ${timeStr} on ${dateStr}. ${sunDirectionHint(ctx.compassHeading || '', hour)}`); } if (contextLines.length) { prompt += `\n\nSpatial/temporal context:\n${contextLines.join('\n')}`; } if (plantName) { prompt += `\n\nThis is a ${plantName}.`; } return prompt; } // ── API call ── async function callMiMo( prompt: string, imageBase64: string, ): Promise<{ content: string; duration: number }> { const start = Date.now(); // Use app's configured token (set via EXPO_PUBLIC_APP_TOKEN in .env) const apiKey = (process.env.EXPO_PUBLIC_APP_TOKEN || '').trim(); const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), requestTimeoutMs); try { const response = await fetch(proxyEndpoint, { method: 'POST', headers: { 'Authorization': `Bearer ${apiKey}`, 'Content-Type': 'application/json', }, body: JSON.stringify({ model: xiaomiModel, messages: [{ role: 'user', content: [ { type: 'text', text: prompt }, { type: 'image_url', image_url: { url: `data:image/jpeg;base64,${imageBase64}` } }, ], }], temperature: 0.2, max_tokens: 1000, response_format: { type: 'json_object' }, }), signal: controller.signal, }); clearTimeout(timeout); const duration = Date.now() - start; if (!response.ok) { return { content: `[HTTP ${response.status}]`, duration }; } const data = await response.json() as { choices?: Array<{ message?: { content?: string } }> }; const content = data.choices?.[0]?.message?.content || '[empty response]'; return { content, duration }; } catch (err) { clearTimeout(timeout); const duration = Date.now() - start; const msg = err instanceof Error ? err.message : 'unknown error'; return { content: `[ERROR: ${msg}]`, duration }; } } // ── JSON extraction ── function extractJsonText(content: string): string { const trimmed = content.trim(); if (trimmed.startsWith('{') && trimmed.endsWith('}')) return trimmed; const fenced = trimmed.match(/```(?:json)?\s*([\s\S]*?)```/i); if (fenced?.[1]) return fenced[1].trim(); const firstBrace = trimmed.indexOf('{'); const lastBrace = trimmed.lastIndexOf('}'); if (firstBrace >= 0 && lastBrace > firstBrace) { return trimmed.slice(firstBrace, lastBrace + 1); } return trimmed; } function parseResponse(raw: string): { summary: string; fields: Record } { try { const jsonText = extractJsonText(raw); const parsed = JSON.parse(jsonText) as Record; const summary = typeof parsed.summary === 'string' ? parsed.summary : ''; const fields = { ...parsed }; delete fields.summary; return { summary, fields }; } catch { return { summary: raw.slice(0, 200), fields: {} }; } } // ── Component ── export function PhotoIntelDebugScreen({ onBack }: { onBack: () => void }) { const [photoUri, setPhotoUri] = useState(null); const [photoType, setPhotoType] = useState('environment'); const [plantName, setPlantName] = useState(''); const [context, setContext] = useState({ compassHeading: '', coarseLat: '', coarseLon: '', timeOfDay: '', indoorOutdoor: '', notes: '', }); const [running, setRunning] = useState(false); const [results, setResults] = useState([]); const [progress, setProgress] = useState(''); // ── Pick photo ── const pickPhoto = async (useCamera: boolean) => { try { let result: ImagePicker.ImagePickerResult; if (useCamera) { const perm = await ImagePicker.requestCameraPermissionsAsync(); if (!perm.granted) { Alert.alert('Camera permission needed'); return; } result = await ImagePicker.launchCameraAsync({ quality: 0.84, allowsEditing: false, }); } else { result = await ImagePicker.launchImageLibraryAsync({ quality: 0.84, allowsEditing: false, mediaTypes: ['images'], }); } if (!result.canceled && result.assets[0]) { // Persist photo to permanent storage so URI survives restarts let permanentUri = result.assets[0].uri; try { const debugDir = `${FileSystem.documentDirectory}pixiesprout-photos/debug/`; const dirInfo = await FileSystem.getInfoAsync(debugDir); if (!dirInfo.exists) { await FileSystem.makeDirectoryAsync(debugDir, { intermediates: true }); } const filename = `debug_${Date.now()}.jpg`; const destUri = `${debugDir}${filename}`; await FileSystem.copyAsync({ from: permanentUri, to: destUri }); permanentUri = destUri; } catch {} setPhotoUri(permanentUri); setResults([]); } } catch (err) { Alert.alert('Error', String(err)); } }; // ── Run comparison ── const runComparison = async () => { if (!photoUri) { Alert.alert('Pick a photo first'); return; } setRunning(true); setResults([]); const allResults: TestResult[] = []; try { // Optimize image setProgress('Optimizing image...'); const optimized = await manipulateAsync( photoUri, [{ resize: { width: maxImageDimension } }], { compress: jpegQuality, format: SaveFormat.JPEG, base64: true }, ); const imageBase64 = optimized.base64 || ''; if (!imageBase64) { Alert.alert('Error', 'Could not optimize image'); setRunning(false); return; } // Test 1: Current prompt setProgress('Testing CURRENT prompt...'); const currentPrompt = buildCurrentPrompt(photoType, plantName || undefined); const currentResult = await callMiMo(currentPrompt, imageBase64); const currentParsed = parseResponse(currentResult.content); allResults.push({ photoType, promptLabel: 'CURRENT (production)', prompt: currentPrompt, rawResponse: currentResult.content, parsedSummary: currentParsed.summary, duration: currentResult.duration, success: !currentResult.content.startsWith('['), }); setResults([...allResults]); // Test 2: Enhanced prompt setProgress('Testing ENHANCED prompt...'); const enhancedPrompt = buildEnhancedPrompt( photoType, context, plantName || undefined, ); const enhancedResult = await callMiMo(enhancedPrompt, imageBase64); const enhancedParsed = parseResponse(enhancedResult.content); allResults.push({ photoType, promptLabel: 'ENHANCED (with context)', prompt: enhancedPrompt, rawResponse: enhancedResult.content, parsedSummary: enhancedParsed.summary, duration: enhancedResult.duration, success: !enhancedResult.content.startsWith('['), }); setResults([...allResults]); setProgress('Done!'); } catch (err) { Alert.alert('Error', String(err)); } finally { setRunning(false); } }; return ( {/* Header */} ← Back 🔬 Photo Intel Debug Compare MiMo's data extraction quality {/* Photo picker */} 📸 Photo {photoUri ? ( ) : ( No photo selected )} pickPhoto(true)}> 📷 Camera pickPhoto(false)}> 🖼️ Gallery {/* Photo type */} 🏷️ Photo Type {(['environment', 'placement', 'setup', 'progress'] as PhotoType[]).map(t => ( setPhotoType(t)} > {t === 'environment' ? '🌍 Env' : t === 'placement' ? '📍 Place' : t === 'setup' ? '🪴 Setup' : '📸 Progress'} ))} {/* Plant name */} 🌿 Plant Name (optional) {/* Context data */} 🧭 Context (optional — enhances the test) Compass direction you're facing {COMPASS_DIRECTIONS.map(dir => ( setContext(c => ({ ...c, compassHeading: c.compassHeading === dir ? '' : dir }))} > {dir} ))} Setting {(['indoor', 'outdoor'] as const).map(t => ( setContext(c => ({ ...c, indoorOutdoor: c.indoorOutdoor === t ? '' : t }))} > {t === 'indoor' ? '🏠 Indoor' : '🌳 Outdoor'} ))} Notes for MiMo (optional) setContext(c => ({ ...c, notes: v }))} placeholder="e.g. 'Garden bed with mixed herbs, south-facing wall'" placeholderTextColor="#999" multiline /> {/* Run button */} {running ? ( {progress} ) : ( 🧪 Run Comparison )} {/* Results */} {results.length > 0 && ( 📊 Results {results.map((r, i) => ( {r.promptLabel} {r.success ? '✅' : '❌'} {r.duration}ms {r.parsedSummary} Raw response: {r.rawResponse} {/* Parsed fields */} {(() => { const parsed = parseResponse(r.rawResponse); const fields = parsed.fields; const keys = Object.keys(fields); if (keys.length === 0) return null; return ( Extracted fields ({keys.length}): {keys.map(k => ( {k}: {JSON.stringify(fields[k])} ))} ); })()} {/* Prompt (collapsible) */} Prompt ({r.prompt.length} chars): {r.prompt} ))} {/* Comparison summary */} {results.length === 2 && ( 🔍 Comparison {(() => { const curr = parseResponse(results[0].rawResponse); const enh = parseResponse(results[1].rawResponse); const currFields = Object.keys(curr.fields).length; const enhFields = Object.keys(enh.fields).length; return ( <> Current: {currFields} field{currFields !== 1 ? 's' : ''} extracted Enhanced: {enhFields} field{enhFields !== 1 ? 's' : ''} extracted Summary quality: current={results[0].parsedSummary.length} chars vs enhanced={results[1].parsedSummary.length} chars Speed: current={results[0].duration}ms vs enhanced={results[1].duration}ms ); })()} )} )} ); } // ── Styles ── const s = StyleSheet.create({ container: { flex: 1, backgroundColor: parchment }, scroll: { paddingHorizontal: 18, paddingTop: 8, paddingBottom: 160 }, header: { marginBottom: 16 }, backBtn: { paddingVertical: 8 }, backText: { fontSize: 16, color: deepLeaf, fontWeight: '600' }, title: { fontSize: 24, fontWeight: '900', color: deepLeaf, marginTop: 4 }, subtitle: { fontSize: 14, color: '#666', marginTop: 2 }, section: { backgroundColor: '#fff', borderRadius: 16, padding: 16, marginBottom: 12, shadowColor: '#000', shadowOpacity: 0.06, shadowRadius: 8, shadowOffset: { width: 0, height: 2 }, elevation: 2, }, sectionTitle: { fontSize: 16, fontWeight: '800', color: deepLeaf, marginBottom: 10 }, preview: { width: '100%', height: 200, borderRadius: 12, marginBottom: 10 }, placeholder: { width: '100%', height: 160, borderRadius: 12, marginBottom: 10, backgroundColor: '#f0f0f0', justifyContent: 'center', alignItems: 'center', }, placeholderText: { color: '#999', fontSize: 14 }, row: { flexDirection: 'row', gap: 10 }, btn: { flex: 1, backgroundColor: freshSprout, borderRadius: 999, paddingVertical: 12, alignItems: 'center', }, btnText: { color: '#fff', fontWeight: '700', fontSize: 14 }, chipRow: { flexDirection: 'row', flexWrap: 'wrap', gap: 8 }, chip: { paddingHorizontal: 14, paddingVertical: 8, borderRadius: 999, backgroundColor: '#f0f0f0', borderWidth: 1, borderColor: softSage, }, chipActive: { backgroundColor: freshSprout, borderColor: freshSprout }, chipText: { fontSize: 13, color: deepLeaf, fontWeight: '600' }, chipTextActive: { color: '#fff' }, chipSmall: { paddingHorizontal: 10, paddingVertical: 6, borderRadius: 999, backgroundColor: '#f0f0f0', borderWidth: 1, borderColor: softSage, }, chipSmallText: { fontSize: 12, color: deepLeaf, fontWeight: '600' }, label: { fontSize: 13, color: '#666', marginTop: 10, marginBottom: 6, fontWeight: '600' }, input: { backgroundColor: '#f8f8f8', borderRadius: 12, paddingHorizontal: 14, paddingVertical: 10, fontSize: 14, color: deepLeaf, borderWidth: 1, borderColor: '#e0e0e0', }, runBtn: { backgroundColor: deepLeaf, borderRadius: 999, paddingVertical: 16, alignItems: 'center', marginBottom: 16, }, runBtnDisabled: { opacity: 0.6 }, runBtnText: { color: '#fff', fontWeight: '800', fontSize: 16 }, runningRow: { flexDirection: 'row', alignItems: 'center', gap: 10 }, resultCard: { backgroundColor: '#f8f8f8', borderRadius: 12, padding: 14, marginBottom: 12, borderWidth: 1, borderColor: '#e8e8e8', }, resultHeader: { flexDirection: 'row', justifyContent: 'space-between', alignItems: 'center', marginBottom: 8 }, resultLabel: { fontSize: 14, fontWeight: '800', color: deepLeaf }, resultMeta: { fontSize: 12, color: '#888' }, resultSummary: { fontSize: 14, color: '#333', lineHeight: 20, marginBottom: 10 }, rawLabel: { fontSize: 11, color: '#999', fontWeight: '600', marginBottom: 4 }, rawScroll: { maxHeight: 120, backgroundColor: '#f0f0f0', borderRadius: 8, padding: 8, marginBottom: 10 }, rawText: { fontSize: 11, color: '#444', fontFamily: Platform.OS === 'ios' ? 'Menlo' : 'monospace' }, fieldsContainer: { marginBottom: 10 }, fieldsTitle: { fontSize: 12, fontWeight: '700', color: deepLeaf, marginBottom: 6 }, fieldRow: { flexDirection: 'row', paddingVertical: 2 }, fieldKey: { fontSize: 12, color: freshSprout, fontWeight: '700', marginRight: 6 }, fieldValue: { fontSize: 12, color: '#444', flex: 1 }, promptLabel: { fontSize: 11, color: '#999', fontWeight: '600', marginBottom: 4 }, promptScroll: { maxHeight: 80, backgroundColor: '#f0f0f0', borderRadius: 8, padding: 8 }, promptText: { fontSize: 10, color: '#666', fontFamily: Platform.OS === 'ios' ? 'Menlo' : 'monospace' }, comparisonCard: { backgroundColor: deepLeaf, borderRadius: 12, padding: 16, marginTop: 4, }, comparisonTitle: { fontSize: 16, fontWeight: '800', color: '#fff', marginBottom: 10 }, comparisonLine: { fontSize: 13, color: softSage, marginBottom: 4, lineHeight: 18 }, });