feat: Add layer-wise texture editing using Gemini AI

backend/server.js

@@ -647,6 +647,77 @@ CRITICAL INSTRUCTIONS:
   }
 });
 
+// Edit Layer (Gemini Flash Image)
+app.post('/api/sparkplug/edit-layer', authenticate, async (req, res) => {
+  const { prompt, imageBase64, mimeType } = req.body;
+  if (!prompt || !imageBase64) return res.status(400).json({ error: 'Missing prompt or image' });
+
+  try {
+    const keys = await getKeysFromProfile(req.user.id);
+    const apiKey = keys.GEMINI_API_KEY || keys.NANO_BANANA_API_KEY;
+    if (!apiKey) return res.status(400).json({ error: 'Gemini API key missing' });
+
+    const baseUrl = 'https://generativelanguage.googleapis.com/v1beta/models/gemini-3.1-flash-image:generateContent';
+    const base64Data = imageBase64.replace(/^data:image\/\w+;base64,/, "");
+    
+    const payload = {
+      contents: [
+        {
+          parts: [
+            { text: `You are an expert texture editor. Edit the following image based on this prompt: "${prompt}". \nCRITICAL INSTRUCTIONS:\n1. Output MUST be an image with the exact same aspect ratio as the input.\n2. Do NOT change the layout or mapping of the texture.\n3. Increase the resolution and quality of the visual elements as requested.` },
+            {
+              inlineData: {
+                mimeType: mimeType || "image/png",
+                data: base64Data
+              }
+            }
+          ]
+        }
+      ],
+      generationConfig: {
+        responseModalities: ["IMAGE"]
+      }
+    };
+
+    const response = await axios.post(`${baseUrl}?key=${apiKey}`, payload, {
+      headers: { 'Content-Type': 'application/json', 'Accept-Encoding': 'gzip,deflate,compress' }
+    });
+
+    let resultBase64 = null;
+    let resultMime = 'image/jpeg';
+    
+    const candidate = response.data?.candidates?.[0];
+    if (candidate?.content?.parts) {
+      for (const part of candidate.content.parts) {
+        if (part.inlineData && part.inlineData.data) {
+          resultBase64 = part.inlineData.data;
+          resultMime = part.inlineData.mimeType || 'image/jpeg';
+          break;
+        }
+      }
+    }
+
+    if (!resultBase64) {
+      throw new Error(`Could not extract base64 image data from Gemini response`);
+    }
+
+    const buffer = Buffer.from(resultBase64, 'base64');
+    const ext = resultMime === 'image/png' ? 'png' : 'jpg';
+    const filename = `layer-edit-${req.user.id}-${Date.now()}.${ext}`;
+    const filepath = path.join(uploadsDir, filename);
+    fs.writeFileSync(filepath, buffer);
+    
+    res.json({ imageUrl: \`/uploads/\${filename}\` });
+  } catch (err) {
+    let errorMsg = err.message;
+    if (err.response && err.response.data) {
+      errorMsg = typeof err.response.data === 'object' ? JSON.stringify(err.response.data) : err.response.data;
+    }
+    console.error("Gemini Edit Layer Error:", errorMsg);
+    res.status(500).json({ error: errorMsg });
+  }
+});
+
 // 3. Generate 3D (Meshy)
 app.post('/api/sparkplug/generate-3d', authenticate, async (req, res) => {
   const { imageUrls } = req.body;

src/components/UVEditor.tsx

@@ -4,6 +4,9 @@ import { Canvas } from '@react-three/fiber';
 import { OrbitControls, Environment } from '@react-three/drei';
 import * as THREE from 'three';
 import { ModelViewer } from './ModelViewer';
+import { editUVLayer } from '../lib/api/apiUtils';
+import { useAuthStore } from '../store/authStore';
+import { Loader2 } from 'lucide-react';
 
 interface Layer {
   id: string;
@@ -45,6 +48,72 @@ export function UVEditor({ onClose, onSave, targetObject, loadedModel, sceneObje
   const fileInputRef = useRef<HTMLInputElement>(null);
   const dragRef = useRef<{ handle: Handle; startX: number; startY: number; layer: Layer } | null>(null);
 
+  const token = useAuthStore(state => state.token);
+  const [aiPrompt, setAiPrompt] = useState("");
+  const [isEditingLayer, setIsEditingLayer] = useState(false);
+  const [aiPreviewUrl, setAiPreviewUrl] = useState<string | null>(null);
+  const [originalLayerUrl, setOriginalLayerUrl] = useState<string | null>(null);
+
+  const handleAiEdit = async (regenerate = false) => {
+    if (!selectedId || !aiPrompt.trim()) return;
+    const layer = layers.find(l => l.id === selectedId);
+    if (!layer || !layer.img) return;
+
+    setIsEditingLayer(true);
+    try {
+      const tempCanvas = document.createElement('canvas');
+      tempCanvas.width = layer.img.width;
+      tempCanvas.height = layer.img.height;
+      const ctx = tempCanvas.getContext('2d');
+      if (ctx) {
+        ctx.drawImage(layer.img, 0, 0);
+        const base64 = tempCanvas.toDataURL('image/png');
+        
+        const newUrl = await editUVLayer(aiPrompt, base64, token || '');
+        
+        if (!originalLayerUrl && !regenerate) {
+          setOriginalLayerUrl(layer.url);
+        }
+        
+        setAiPreviewUrl(newUrl);
+        
+        const img = new Image();
+        img.crossOrigin = "anonymous";
+        img.onload = () => {
+          setLayers(ls => ls.map(l => l.id === selectedId ? { ...l, url: newUrl, img } : l));
+          setIsEditingLayer(false);
+        };
+        img.src = newUrl;
+        return;
+      }
+    } catch (e: any) {
+      alert("AI Edit failed: " + e.message);
+    }
+    setIsEditingLayer(false);
+  };
+
+  const handleApproveEdit = () => {
+    setAiPreviewUrl(null);
+    setOriginalLayerUrl(null);
+    setAiPrompt("");
+  };
+
+  const handleRejectEdit = () => {
+    if (selectedId && originalLayerUrl) {
+      const img = new Image();
+      img.crossOrigin = "anonymous";
+      img.onload = () => {
+        setLayers(ls => ls.map(l => l.id === selectedId ? { ...l, url: originalLayerUrl, img } : l));
+        setAiPreviewUrl(null);
+        setOriginalLayerUrl(null);
+      };
+      img.src = originalLayerUrl;
+    } else {
+      setAiPreviewUrl(null);
+      setOriginalLayerUrl(null);
+    }
+  };
+
   useEffect(() => {
     const saved = targetObject?.materialProps?.uvLayers;
     if (saved?.length) {
@@ -127,298 +196,6 @@ export function UVEditor({ onClose, onSave, targetObject, loadedModel, sceneObje
       console.error("Canvas render error (likely CORS taint):", e);
       return "";
     }
-  }, [canvasW, canvasH]);
-
-  useEffect(() => {
-    setPreviewUrl(renderCanvas(layers, bgTransparent, bgColor));
-  }, [layers, bgTransparent, bgColor]);
-
-  const handleFileUpload = (e: React.ChangeEvent<HTMLInputElement>) => {
-    const file = e.target.files?.[0]; if (!file) return;
-    const reader = new FileReader();
-    reader.onload = ev => {
-      const url = ev.target?.result as string;
-      const img = new Image();
-      img.onload = () => {
-        const nl: Layer = { id: `l-${Date.now()}`, url, img, x: canvasW / 2, y: canvasH / 2, scaleX: 1, scaleY: 1, rotation: 0, cropL: 0, cropT: 0, cropR: 0, cropB: 0, visible: true };
-        setLayers(p => [...p, nl]);
-        setSelectedId(nl.id);
-      };
-      img.src = url;
-    };
-    reader.readAsDataURL(file);
-    e.target.value = '';
-  };
-
-  const updateLayer = (id: string, u: Partial<Layer>) => setLayers(p => p.map(l => l.id === id ? { ...l, ...u } : l));
-  const selectedLayer = layers.find(l => l.id === selectedId) ?? null;
-
-  const getSF = () => {
-    const r = containerRef.current?.getBoundingClientRect();
-    return r ? canvasW / r.width : 1;
-  };
-
-  const onHandleDown = (e: React.MouseEvent, handle: Handle, layer: Layer) => {
-    e.stopPropagation();
-    setSelectedId(layer.id);
-    dragRef.current = { handle, startX: e.clientX, startY: e.clientY, layer: { ...layer } };
-    e.preventDefault();
-  };
-
-  // Snap helper: snap value to nearest multiple of snapSize
-  const snap = (v: number, snapSize: number) => Math.round(v / snapSize) * snapSize;
-
-  const onMouseMove = (e: React.MouseEvent) => {
-    const d = dragRef.current;
-    if (!d) return;
-    const sf = getSF();
-    const dx = (e.clientX - d.startX) * sf;
-    const dy = (e.clientY - d.startY) * sf;
-    const isShift = e.shiftKey;
-    const SNAP_PX = 16; // grid snap in canvas units
-
-    if (d.handle === 'move') {
-      let nx = d.layer.x + dx;
-      let ny = d.layer.y + dy;
-      // Clamp to canvas bounds with soft margin
-      nx = Math.max(0, Math.min(canvasW, nx));
-      ny = Math.max(0, Math.min(canvasH, ny));
-      // Hold Shift to snap to grid
-      if (isShift) { nx = snap(nx, SNAP_PX); ny = snap(ny, SNAP_PX); }
-      // Center snapping: snap near canvas center
-      if (Math.abs(nx - canvasW / 2) < SNAP_PX) nx = canvasW / 2;
-      if (Math.abs(ny - canvasH / 2) < SNAP_PX) ny = canvasH / 2;
-      updateLayer(d.layer.id, { x: nx, y: ny });
-    } else if (d.handle === 'rotate') {
-      let rot = d.layer.rotation + dx / 2;
-      // Snap rotation to 15° increments when Shift held
-      if (isShift) rot = snap(rot, 15);
-      updateLayer(d.layer.id, { rotation: rot });
-    } else if (d.handle.startsWith('scale')) {
-      if (isShift) {
-        // Shift held: stretch X and Y independently
-        const sx = Math.max(0.05, d.layer.scaleX + dx / 200);
-        const sy = Math.max(0.05, d.layer.scaleY + dy / 200);
-        updateLayer(d.layer.id, { scaleX: sx, scaleY: sy });
-      } else {
-        // No shift: uniform scale driven by the larger axis
-        const delta = (Math.abs(dx) > Math.abs(dy) ? dx : dy) / 200;
-        const sx = Math.max(0.05, d.layer.scaleX + delta);
-        const sy = Math.max(0.05, d.layer.scaleY + delta);
-        updateLayer(d.layer.id, { scaleX: sx, scaleY: sy });
-      }
-    }
-  };
-
-  const onMouseUp = () => { dragRef.current = null; };
-
-  const handleSave = () => {
-    if (previewUrl) {
-      const serialized = layers.map(({ img, ...rest }) => rest);
-      onSave(previewUrl, mappingType, serialized, bgColor, bgTransparent);
-    }
-    onClose();
-  };
-
-  const PreviewModel = () => {
-    const mockObjs = useMemo(() => {
-      if (!sceneObjects) return [];
-      return sceneObjects.map(obj => obj.id === targetObject?.id
-        ? { ...obj, materialProps: { ...obj.materialProps, map: previewUrl, mappingType } }
-        : obj
-      );
-    }, [sceneObjects, previewUrl, mappingType]);
-    const cloned = useMemo(() => loadedModel?.clone() ?? null, [loadedModel]);
-    if (!targetObject) return <mesh><boxGeometry /><meshStandardMaterial /></mesh>;
-    if (targetObject.type === 'cube') {
-      const tex = previewUrl ? (() => { const t = new THREE.TextureLoader().load(previewUrl); t.colorSpace = THREE.SRGBColorSpace; t.flipY = false; return t; })() : null;
-      return <mesh><boxGeometry args={[1, 1, 1]} /><meshStandardMaterial map={tex ?? undefined} /></mesh>;
-    }
-    return <ModelViewer object={cloned} sceneObjects={mockObjs} />;
-  };
-
-  // Bounding box overlay rendered on canvas div
-  const BoundingBox = ({ layer }: { layer: Layer }) => {
-    const iw = (layer.img?.width ?? 200) * (1 - layer.cropL - layer.cropR);
-    const ih = (layer.img?.height ?? 200) * (1 - layer.cropT - layer.cropB);
-    const rect = containerRef.current?.getBoundingClientRect();
-    if (!rect) return null;
-    
-    const w = (iw * layer.scaleX / canvasW) * 100;
-    const h = (ih * layer.scaleY / canvasH) * 100;
-    const lx = (layer.x / canvasW) * 100;
-    const ly = (layer.y / canvasH) * 100;
-
-    const cornerStyle = (cx: string, cy: string, cursor: string): React.CSSProperties => ({
-      position: 'absolute', left: cx, top: cy, width: 10, height: 10,
-      background: '#ffffff', border: '2px solid #3b82f6', borderRadius: 2,
-      transform: 'translate(-50%,-50%)', cursor, zIndex: 10, touchAction: 'none'
-    });
-    const edgeStyle = (cx: string, cy: string, cursor: string): React.CSSProperties => ({
-      position: 'absolute', left: cx, top: cy, width: 8, height: 8,
-      background: '#3b82f6', borderRadius: '50%',
-      transform: 'translate(-50%,-50%)', cursor, zIndex: 10
-    });
-
-    return (
-      <div style={{
-        position: 'absolute',
-        left: `${lx}%`, top: `${ly}%`,
-        width: `${w}%`, height: `${h}%`,
-        transform: `translate(-50%,-50%) rotate(${layer.rotation}deg)`,
-        border: '1.5px solid #3b82f6',
-        pointerEvents: 'none',
-        zIndex: 5,
-      }}>
-        {/* Corners - scale */}
-        {(['NW','NE','SW','SE'] as const).map(c => (
-          <div key={c}
-            style={{
-              ...cornerStyle(
-                c.includes('W') ? '0%' : '100%',
-                c.includes('N') ? '0%' : '100%',
-                c === 'NW' || c === 'SE' ? 'nwse-resize' : 'nesw-resize'
-              ), pointerEvents: 'all'
-            }}
-            onMouseDown={e => onHandleDown(e, `scale${c}` as Handle, layer)}
-          />
-        ))}
-        {/* Edge midpoints - scale */}
-        {[
-          { id: 'N', cx: '50%', cy: '0%', cur: 'ns-resize' },
-          { id: 'S', cx: '50%', cy: '100%', cur: 'ns-resize' },
-          { id: 'E', cx: '100%', cy: '50%', cur: 'ew-resize' },
-          { id: 'W', cx: '0%', cy: '50%', cur: 'ew-resize' },
-        ].map(({ id, cx, cy, cur }) => (
-          <div key={id} style={{ ...edgeStyle(cx, cy, cur), pointerEvents: 'all' }}
-            onMouseDown={e => onHandleDown(e, `scale${id}` as Handle, layer)} />
-        ))}
-        {/* Rotate handle - top center above box */}
-        <div style={{
-          position: 'absolute', left: '50%', top: -28,
-          transform: 'translateX(-50%)',
-          width: 14, height: 14,
-          background: '#f59e0b', border: '2px solid #fff', borderRadius: '50%',
-          cursor: 'grab', zIndex: 10, pointerEvents: 'all',
-        }} onMouseDown={e => onHandleDown(e, 'rotate', layer)} title="Rotate" />
-        {/* Line from box to rotate handle */}
-        <div style={{ position: 'absolute', left: '50%', top: -20, width: 2, height: 20, background: '#3b82f6', transform: 'translateX(-50%)', pointerEvents: 'none' }} />
-      </div>
-    );
-  };
-
-  return (
-    <div className="fixed inset-0 z-[200] flex flex-col text-zinc-100" style={{ fontFamily: 'Inter,sans-serif', background: '#0f0f13' }}>
-      {/* Top Bar */}
-      <div className="h-11 border-b border-zinc-800 bg-zinc-900/95 flex items-center justify-between px-4 flex-shrink-0 backdrop-blur">
-        <div className="flex items-center gap-3">
-          <button onClick={onClose} className="p-1.5 hover:bg-zinc-800 rounded-lg text-zinc-400 hover:text-white transition-colors">
-            <X size={16} />
-          </button>
-          <div className="w-px h-4 bg-zinc-700" />
-          <span className="text-sm font-semibold">UV Map</span>
-        </div>
-        <div className="flex items-center gap-3">
-          <label className="flex items-center gap-1.5 text-xs text-zinc-400 cursor-pointer select-none">
-            <input type="checkbox" checked={bgTransparent} onChange={e => setBgTransparent(e.target.checked)} className="accent-blue-500 w-3 h-3" />
-            Transparent
-          </label>
-          {!bgTransparent && (
-            <label className="relative flex items-center gap-1.5 cursor-pointer">
-              <div className="w-5 h-5 rounded border border-zinc-600 overflow-hidden relative">
-                <div className="absolute inset-0" style={{ background: bgColor }} />
-                <input type="color" value={bgColor} onChange={e => setBgColor(e.target.value)} className="opacity-0 absolute inset-0 w-full h-full cursor-pointer" />
-              </div>
-              <span className="text-[10px] font-mono text-zinc-400">{bgColor}</span>
-            </label>
-          )}
-          <button onClick={handleSave} className="px-4 py-1.5 bg-blue-500 hover:bg-blue-600 rounded-lg text-sm font-semibold transition-colors">
-            Apply to Model
-          </button>
-        </div>
-      </div>
-
-      <div className="flex-1 flex overflow-hidden">
-        {/* Left sidebar */}
-        <div className="w-60 bg-zinc-900 border-r border-zinc-800 flex flex-col overflow-y-auto flex-shrink-0">
-          <div className="p-4 flex flex-col gap-3">
-            <input type="file" ref={fileInputRef} className="hidden" accept=".jpg,.jpeg,.png,.webp,.svg" onChange={handleFileUpload} />
-            <button onClick={() => fileInputRef.current?.click()}
-              className="w-full py-2 bg-zinc-800 hover:bg-zinc-700 rounded-lg text-sm font-semibold flex items-center justify-center gap-2 border border-zinc-700 transition-colors">
-              <Upload size={14} /> Add Image
-            </button>
-            <div className="h-px bg-zinc-800" />
-            <p className="text-[10px] font-bold text-zinc-500 uppercase tracking-widest">Layers</p>
-            {layers.length === 0 && (
-              <div className="p-5 text-center text-zinc-600 text-xs border-2 border-dashed border-zinc-800 rounded-xl">Upload an image to start</div>
-            )}
-            {[...layers].reverse().map((layer, i) => (
-              <div key={layer.id} onClick={() => setSelectedId(layer.id)}
-                className={`flex items-center gap-2 p-2 rounded-lg border cursor-pointer transition-all ${selectedId === layer.id ? 'border-blue-500 bg-blue-500/10' : 'border-zinc-800 hover:border-zinc-600 bg-zinc-800/20'} ${!layer.visible ? 'opacity-40' : ''}`}>
-                <img src={layer.url} alt="" className="w-8 h-8 object-contain rounded bg-zinc-900 border border-zinc-700 flex-shrink-0" />
-                <span className="text-xs text-zinc-300 flex-1 truncate">Layer {layers.length - i}</span>
-                <button onClick={e => { e.stopPropagation(); updateLayer(layer.id, { visible: !layer.visible }); }}
-                  className="p-1 text-zinc-600 hover:text-blue-400 rounded" title={layer.visible ? 'Hide layer' : 'Show layer'}>
-                  {layer.visible !== false ? <Eye size={11} /> : <EyeOff size={11} />}
-                </button>
-                <button onClick={e => { e.stopPropagation(); setLayers(ls => ls.filter(l => l.id !== layer.id)); if (selectedId === layer.id) setSelectedId(null); }}
-                  className="p-1 text-zinc-600 hover:text-red-400 rounded">
-                  <Trash2 size={11} />
-                </button>
-              </div>
-            ))}
-
-            {/* Layer props */}
-            {selectedLayer && (
-              <div className="mt-1 flex flex-col gap-2 p-3 rounded-xl border border-zinc-800 bg-zinc-800/20">
-                <p className="text-[10px] font-bold text-zinc-500 uppercase tracking-widest">Transform</p>
-                <div className="grid grid-cols-2 gap-2">
-                  {(['x', 'y'] as const).map(ax => (
-                    <div key={ax}>
-                      <p className="text-[9px] text-zinc-500 uppercase mb-0.5">{ax}</p>
-                      <input type="number" step="1" value={Math.round(selectedLayer[ax])}
-                        onChange={e => updateLayer(selectedLayer.id, { [ax]: +e.target.value })}
-                        className="w-full bg-zinc-900 border border-zinc-700 rounded px-2 py-1 text-[11px] text-blue-400 font-mono outline-none focus:border-blue-500" />
-                    </div>
-                  ))}
-                </div>
-                <div className="grid grid-cols-2 gap-2">
-                  {(['scaleX', 'scaleY'] as const).map(ax => (
-                    <div key={ax}>
-                      <p className="text-[9px] text-zinc-500 uppercase mb-0.5">{ax === 'scaleX' ? 'Scl X' : 'Scl Y'}</p>
-                      <input type="number" step="0.01" min="0.01" value={selectedLayer[ax].toFixed(2)}
-                        onChange={e => updateLayer(selectedLayer.id, { [ax]: Math.max(0.01, +e.target.value) })}
-                        className="w-full bg-zinc-900 border border-zinc-700 rounded px-2 py-1 text-[11px] text-blue-400 font-mono outline-none focus:border-blue-500" />
-                    </div>
-                  ))}
-                </div>
-                <div>
-                  <div className="flex justify-between text-[9px] text-zinc-500 uppercase mb-1">
-                    <span>Rotation</span><span className="text-blue-400 font-mono">{Math.round(selectedLayer.rotation)}°</span>
-                  </div>
-                  <input type="range" min="-180" max="180" step="1" value={selectedLayer.rotation}
-                    onChange={e => updateLayer(selectedLayer.id, { rotation: +e.target.value })}
-                    className="w-full h-1 bg-zinc-700 rounded appearance-none cursor-pointer accent-blue-500" />
-                </div>
-
-                <div className="h-px bg-zinc-700 my-1" />
-                <p className="text-[10px] font-bold text-zinc-500 uppercase tracking-widest">Crop</p>
-                {(['cropL', 'cropT', 'cropR', 'cropB'] as const).map(s => (
-                  <div key={s} className="flex items-center gap-2">
-                    <span className="text-[9px] text-zinc-500 w-6 uppercase">{s.slice(4)}</span>
-                    <input type="range" min="0" max="0.49" step="0.01" value={selectedLayer[s]}
-                      onChange={e => updateLayer(selectedLayer.id, { [s]: +e.target.value })}
-                      className="flex-1 h-1 bg-zinc-700 rounded appearance-none cursor-pointer accent-orange-500" />
-                    <span className="text-[9px] font-mono text-orange-400 w-6 text-right">{Math.round(selectedLayer[s] * 100)}%</span>
-                  </div>
-                ))}
-              </div>
-            )}
-          </div>
-        </div>
-
-        {/* Canvas area */}
-        <div className="flex-1 flex items-center justify-center overflow-hidden relative"
           style={{ background: 'radial-gradient(ellipse at center, #1a1a2e 0%, #0f0f13 100%)' }}
           onMouseMove={onMouseMove}
           onMouseUp={onMouseUp}

src/lib/api/apiUtils.ts

@@ -79,3 +79,33 @@ export async function pollStatus<T>(
 
   throw new Error("Polling timeout exceeded.");
 }
+
+export async function editUVLayer(prompt: string, imageBase64: string, token: string): Promise<string> {
+  // Extract mime type if present in the data URI
+  let mimeType = 'image/png';
+  if (imageBase64.startsWith('data:')) {
+    const match = imageBase64.match(/^data:([^;]+);/);
+    if (match) {
+      mimeType = match[1];
+    }
+  }
+
+  const response = await fetch('/api/sparkplug/edit-layer', {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      'Authorization': `Bearer ${token}`
+    },
+    body: JSON.stringify({ prompt, imageBase64, mimeType })
+  });
+
+  if (!response.ok) {
+    const errorData = await response.json().catch(() => ({}));
+    throw new Error(errorData.error || `HTTP error! status: ${response.status}`);
+  }
+
+  const data = await response.json();
+  if (data.error) throw new Error(data.error);
+  
+  return data.imageUrl;
+}