diff --git a/backend/package-lock.json b/backend/package-lock.json index fd217c5..3893442 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -8,6 +8,8 @@ "name": "titan3d-backend", "version": "1.0.0", "dependencies": { + "@google/generative-ai": "^0.24.1", + "axios": "^1.20.0", "bcrypt": "^5.1.1", "cors": "^2.8.5", "dotenv": "^17.4.2", @@ -15,10 +17,20 @@ "jsonwebtoken": "^9.0.2", "multer": "^1.4.5-lts.1", "nodemailer": "^9.0.5", + "pdf-parse": "^2.4.5", "pg": "^8.23.0", "uuid": "^9.0.1" } }, + "node_modules/@google/generative-ai": { + "version": "0.24.1", + "resolved": "https://registry.npmjs.org/@google/generative-ai/-/generative-ai-0.24.1.tgz", + "integrity": "sha512-MqO+MLfM6kjxcKoy0p1wRzG3b4ZZXtPI+z2IE26UogS2Cm/XHO+7gGRBh6gcJsOiIVoH93UwKvW4HdgiOZCy9Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@mapbox/node-pre-gyp": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.11.tgz", @@ -39,6 +51,190 @@ "node-pre-gyp": "bin/node-pre-gyp" } }, + "node_modules/@napi-rs/canvas": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.80.tgz", + "integrity": "sha512-DxuT1ClnIPts1kQx8FBmkk4BQDTfI5kIzywAaMjQSXfNnra5UFU9PwurXrl+Je3bJ6BGsp/zmshVVFbCmyI+ww==", + "license": "MIT", + "workspaces": [ + "e2e/*" + ], + "engines": { + "node": ">= 10" + }, + "optionalDependencies": { + "@napi-rs/canvas-android-arm64": "0.1.80", + "@napi-rs/canvas-darwin-arm64": "0.1.80", + "@napi-rs/canvas-darwin-x64": "0.1.80", + "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.80", + "@napi-rs/canvas-linux-arm64-gnu": "0.1.80", + "@napi-rs/canvas-linux-arm64-musl": "0.1.80", + "@napi-rs/canvas-linux-riscv64-gnu": "0.1.80", + "@napi-rs/canvas-linux-x64-gnu": "0.1.80", + "@napi-rs/canvas-linux-x64-musl": "0.1.80", + "@napi-rs/canvas-win32-x64-msvc": "0.1.80" + } + }, + "node_modules/@napi-rs/canvas-android-arm64": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.80.tgz", + "integrity": "sha512-sk7xhN/MoXeuExlggf91pNziBxLPVUqF2CAVnB57KLG/pz7+U5TKG8eXdc3pm0d7Od0WreB6ZKLj37sX9muGOQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-darwin-arm64": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.80.tgz", + "integrity": "sha512-O64APRTXRUiAz0P8gErkfEr3lipLJgM6pjATwavZ22ebhjYl/SUbpgM0xcWPQBNMP1n29afAC/Us5PX1vg+JNQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-darwin-x64": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.80.tgz", + "integrity": "sha512-FqqSU7qFce0Cp3pwnTjVkKjjOtxMqRe6lmINxpIZYaZNnVI0H5FtsaraZJ36SiTHNjZlUB69/HhxNDT1Aaa9vA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-arm-gnueabihf": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.80.tgz", + "integrity": "sha512-eyWz0ddBDQc7/JbAtY4OtZ5SpK8tR4JsCYEZjCE3dI8pqoWUC8oMwYSBGCYfsx2w47cQgQCgMVRVTFiiO38hHQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-gnu": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.80.tgz", + "integrity": "sha512-qwA63t8A86bnxhuA/GwOkK3jvb+XTQaTiVML0vAWoHyoZYTjNs7BzoOONDgTnNtr8/yHrq64XXzUoLqDzU+Uuw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-musl": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.80.tgz", + "integrity": "sha512-1XbCOz/ymhj24lFaIXtWnwv/6eFHXDrjP0jYkc6iHQ9q8oXKzUX1Lc6bu+wuGiLhGh2GS/2JlfORC5ZcXimRcg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-riscv64-gnu": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.80.tgz", + "integrity": "sha512-XTzR125w5ZMs0lJcxRlS1K3P5RaZ9RmUsPtd1uGt+EfDyYMu4c6SEROYsxyatbbu/2+lPe7MPHOO/0a0x7L/gw==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-gnu": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.80.tgz", + "integrity": "sha512-BeXAmhKg1kX3UCrJsYbdQd3hIMDH/K6HnP/pG2LuITaXhXBiNdh//TVVVVCBbJzVQaV5gK/4ZOCMrQW9mvuTqA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-musl": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.80.tgz", + "integrity": "sha512-x0XvZWdHbkgdgucJsRxprX/4o4sEed7qo9rCQA9ugiS9qE2QvP0RIiEugtZhfLH3cyI+jIRFJHV4Fuz+1BHHMg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-win32-x64-msvc": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.80.tgz", + "integrity": "sha512-Z8jPsM6df5V8B1HrCHB05+bDiCxjE9QA//3YrkKIdVDEwn5RKaqOxCJDRJkl48cJbylcrJbW4HxZbTte8juuPg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, "node_modules/abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -148,6 +344,24 @@ "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", "license": "MIT" }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, + "node_modules/axios": { + "version": "1.20.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.20.0.tgz", + "integrity": "sha512-r8aOh8j9cGKpgQAqpzrUHnSIc6a59Y3Xf/cv8sy1DrHCkZHzQGEuoq1tARk6qSyDdtQGSDgpb9kFlruzPvrgwg==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.16.0", + "form-data": "^4.0.6", + "https-proxy-agent": "^5.0.1", + "proxy-from-env": "^2.1.0" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -281,6 +495,18 @@ "color-support": "bin.js" } }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -376,6 +602,15 @@ "ms": "2.0.0" } }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/delegates": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", @@ -521,6 +756,21 @@ "node": ">= 0.4" } }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", @@ -600,6 +850,42 @@ "node": ">= 0.8" } }, + "node_modules/follow-redirects": { + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz", + "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz", + "integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.4", + "mime-types": "^2.1.35" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -748,6 +1034,21 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/has-unicode": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", @@ -1280,6 +1581,38 @@ "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==", "license": "MIT" }, + "node_modules/pdf-parse": { + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/pdf-parse/-/pdf-parse-2.4.5.tgz", + "integrity": "sha512-mHU89HGh7v+4u2ubfnevJ03lmPgQ5WU4CxAVmTSh/sxVTEDYd1er/dKS/A6vg77NX47KTEoihq8jZBLr8Cxuwg==", + "license": "Apache-2.0", + "dependencies": { + "@napi-rs/canvas": "0.1.80", + "pdfjs-dist": "5.4.296" + }, + "bin": { + "pdf-parse": "bin/cli.mjs" + }, + "engines": { + "node": ">=20.16.0 <21 || >=22.3.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/mehmet-kozan" + } + }, + "node_modules/pdfjs-dist": { + "version": "5.4.296", + "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-5.4.296.tgz", + "integrity": "sha512-DlOzet0HO7OEnmUmB6wWGJrrdvbyJKftI1bhMitK7O2N8W2gc757yyYBbINy9IDafXAV9wmKr9t7xsTaNKRG5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=20.16.0 || >=22.3.0" + }, + "optionalDependencies": { + "@napi-rs/canvas": "^0.1.80" + } + }, "node_modules/pg": { "version": "8.23.0", "resolved": "https://registry.npmjs.org/pg/-/pg-8.23.0.tgz", @@ -1427,6 +1760,15 @@ "node": ">= 0.10" } }, + "node_modules/proxy-from-env": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-2.1.0.tgz", + "integrity": "sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, "node_modules/qs": { "version": "6.15.3", "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", diff --git a/backend/package.json b/backend/package.json index 163ae1c..c8c60e0 100644 --- a/backend/package.json +++ b/backend/package.json @@ -7,6 +7,8 @@ "start": "node server.js" }, "dependencies": { + "@google/generative-ai": "^0.24.1", + "axios": "^1.20.0", "bcrypt": "^5.1.1", "cors": "^2.8.5", "dotenv": "^17.4.2", @@ -14,6 +16,7 @@ "jsonwebtoken": "^9.0.2", "multer": "^1.4.5-lts.1", "nodemailer": "^9.0.5", + "pdf-parse": "^2.4.5", "pg": "^8.23.0", "uuid": "^9.0.1" } diff --git a/backend/server.js b/backend/server.js index 84a6d22..6da2c31 100644 --- a/backend/server.js +++ b/backend/server.js @@ -7,6 +7,9 @@ const { v4: uuidv4 } = require('uuid'); const multer = require('multer'); const path = require('path'); const fs = require('fs'); +const pdfParse = require('pdf-parse'); +const { GoogleGenerativeAI } = require('@google/generative-ai'); +const axios = require('axios'); const { db, run, get, all } = require('./database'); const { sendMail } = require('./mail'); @@ -305,6 +308,176 @@ app.post('/api/upload/complete', authenticate, (req, res) => { } }); +// ========================================== +// SPARK PLUG WORKFLOW APIS +// ========================================== + +const getKeysFromProfile = async (userId) => { + const profile = await get('SELECT api_keys FROM profiles WHERE id = ?', [userId]); + if (profile && profile.api_keys) { + try { + return JSON.parse(profile.api_keys); + } catch(e) {} + } + return {}; +}; + +// 1. Extract PDF (Gemini) +app.post('/api/sparkplug/extract', authenticate, upload.single('file'), async (req, res) => { + if (!req.file) return res.status(400).json({ error: 'No file uploaded' }); + + try { + const keys = await getKeysFromProfile(req.user.id); + const geminiKey = keys.GEMINI_API_KEY; + if (!geminiKey) return res.status(400).json({ error: 'Gemini API key missing' }); + + // Parse PDF + const dataBuffer = fs.readFileSync(req.file.path); + const pdfData = await pdfParse(dataBuffer); + const textContent = pdfData.text; + + // Call Gemini to extract prompt + const genAI = new GoogleGenerativeAI(geminiKey); + const model = genAI.getGenerativeModel({ model: "gemini-1.5-flash" }); + + const prompt = ` + You are an expert product designer. Read the following text from a product spec PDF and create a highly detailed, concise visual design prompt for an image generation AI. + Focus on the object's shape, color, materials, packaging details, and label content. Do not include background details. + + TEXT: + ${textContent.substring(0, 30000)} // Limit to avoid context length issues if massive + `; + + const result = await model.generateContent(prompt); + const response = await result.response; + const extractedPrompt = response.text(); + + res.json({ prompt: extractedPrompt }); + } catch (err) { + res.status(500).json({ error: err.message }); + } finally { + if (req.file) { + fs.unlinkSync(req.file.path); // cleanup uploaded PDF + } + } +}); + +// 2. Generate Images (Nano Banana / Gemini Imagen) +app.post('/api/sparkplug/generate-images', authenticate, async (req, res) => { + const { prompt } = req.body; + if (!prompt) return res.status(400).json({ error: 'No prompt provided' }); + + try { + const keys = await getKeysFromProfile(req.user.id); + const apiKey = keys.NANO_BANANA_API_KEY || keys.GEMINI_API_KEY; + if (!apiKey) return res.status(400).json({ error: 'Nano Banana (Gemini) API key missing' }); + + // We will use axios to call the Gemini Imagen REST API + const baseUrl = 'https://generativelanguage.googleapis.com/v1beta/models/imagen-3.0-generate-001:predict'; + + const views = ['Front view', 'Back view', 'Left side view', 'Right side view']; + const imageUrls = []; + + // Run parallel generation for all 4 views + const promises = views.map(async (view, index) => { + const fullPrompt = `${prompt}. ${view}, isolated on a pure white background, studio lighting.`; + + const payload = { + instances: [ + { + prompt: fullPrompt + } + ], + parameters: { + sampleCount: 1, + outputOptions: { mimeType: 'image/png' } + } + }; + + const response = await axios.post(`${baseUrl}?key=${apiKey}`, payload, { + headers: { 'Content-Type': 'application/json' } + }); + + const base64Image = response.data.predictions[0].bytesBase64Encoded; + const buffer = Buffer.from(base64Image, 'base64'); + + const filename = `sparkplug-${req.user.id}-${Date.now()}-${index}.png`; + const filepath = path.join(uploadsDir, filename); + fs.writeFileSync(filepath, buffer); + + return `/uploads/${filename}`; + }); + + const generatedUrls = await Promise.all(promises); + res.json({ images: generatedUrls }); + } catch (err) { + // Gemini HTTP errors usually have response.data.error + const errorMsg = err.response?.data?.error?.message || err.message; + res.status(500).json({ error: errorMsg }); + } +}); + +// 3. Generate 3D (Meshy) +app.post('/api/sparkplug/generate-3d', authenticate, async (req, res) => { + const { imageUrls } = req.body; + if (!imageUrls || imageUrls.length === 0) return res.status(400).json({ error: 'No images provided' }); + + try { + const keys = await getKeysFromProfile(req.user.id); + const meshyKey = keys.MESHY_API_KEY; + if (!meshyKey) return res.status(400).json({ error: 'Meshy API key missing' }); + + // Assuming imageUrls are local paths like /uploads/... + // Meshy requires base64 Data URIs if the images are not publicly accessible URLs. + // Our local URLs are not publicly accessible to Meshy's servers! We MUST convert to base64. + + const base64Images = imageUrls.map(url => { + // Extract filename from URL + const filename = url.replace('/uploads/', ''); + const filepath = path.join(uploadsDir, filename); + const buffer = fs.readFileSync(filepath); + return `data:image/png;base64,${buffer.toString('base64')}`; + }); + + const response = await axios.post('https://api.meshy.ai/openapi/v1/multi-image-to-3d', { + image_urls: base64Images, + enable_pbr: true + }, { + headers: { + 'Authorization': `Bearer ${meshyKey}`, + 'Content-Type': 'application/json' + } + }); + + res.json({ taskId: response.data.result }); + } catch (err) { + const errorMsg = err.response?.data?.message || err.message; + res.status(500).json({ error: errorMsg }); + } +}); + +// 4. Check 3D Status (Meshy) +app.get('/api/sparkplug/status-3d/:taskId', authenticate, async (req, res) => { + const { taskId } = req.params; + + try { + const keys = await getKeysFromProfile(req.user.id); + const meshyKey = keys.MESHY_API_KEY; + if (!meshyKey) return res.status(400).json({ error: 'Meshy API key missing' }); + + const response = await axios.get(`https://api.meshy.ai/openapi/v1/multi-image-to-3d/${taskId}`, { + headers: { + 'Authorization': `Bearer ${meshyKey}` + } + }); + + res.json(response.data); + } catch (err) { + const errorMsg = err.response?.data?.message || err.message; + res.status(500).json({ error: errorMsg }); + } +}); + const PORT = process.env.PORT || 3005; app.listen(PORT, () => { console.log(`Backend server running on port ${PORT}`); diff --git a/src/lib/api.ts b/src/lib/api.ts index 1508a8a..45644a0 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -192,5 +192,57 @@ export const api = { if (!res.ok) throw new Error(data.error); return data.url; } + }, + + // ========================================== + // SPARK PLUG APIS + // ========================================== + + extractPDF: async (file: File) => { + const formData = new FormData(); + formData.append('file', file); + + const res = await fetch(`${API_URL}/sparkplug/extract`, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${localStorage.getItem('auth_token')}` + }, + body: formData + }); + const data = await res.json(); + if (!res.ok) throw new Error(data.error); + return data.prompt; + }, + + generateImages: async (prompt: string) => { + const res = await fetch(`${API_URL}/sparkplug/generate-images`, { + method: 'POST', + headers: getHeaders(), + body: JSON.stringify({ prompt }) + }); + const data = await res.json(); + if (!res.ok) throw new Error(data.error); + return data.images; // Array of 4 image URLs + }, + + generate3D: async (imageUrls: string[]) => { + const res = await fetch(`${API_URL}/sparkplug/generate-3d`, { + method: 'POST', + headers: getHeaders(), + body: JSON.stringify({ imageUrls }) + }); + const data = await res.json(); + if (!res.ok) throw new Error(data.error); + return data.taskId; + }, + + get3DStatus: async (taskId: string) => { + const res = await fetch(`${API_URL}/sparkplug/status-3d/${taskId}`, { + method: 'GET', + headers: getHeaders() + }); + const data = await res.json(); + if (!res.ok) throw new Error(data.error); + return data; // { status: "SUCCEEDED"|"PENDING", model_urls: { glb: "..." }, ... } } }; diff --git a/src/pages/SparkPlug.tsx b/src/pages/SparkPlug.tsx index ce7acba..5015d83 100644 --- a/src/pages/SparkPlug.tsx +++ b/src/pages/SparkPlug.tsx @@ -27,6 +27,7 @@ export default function SparkPlug() { // Step 4: 3D Model const [modelProgress, setModelProgress] = useState(0); + const [generatedGlbUrl, setGeneratedGlbUrl] = useState(null); const handleFileUpload = async (e: React.ChangeEvent) => { if (e.target.files && e.target.files[0]) { @@ -49,17 +50,14 @@ export default function SparkPlug() { setIsLoading(false); return; } - - // We would normally send the PDF to the backend or use Gemini SDK directly here. - // Mocking the extraction for now based on the Indica Shampoo PDF context: - await new Promise(r => setTimeout(r, 2000)); - - setExtractedPrompt("Product: Indica Easy Herbal Shield Colour Shampoo\nType: Sachet/Packet\nColor Palette: Green, White\nKey Elements: Woman with black hair, Amla & Aloe illustration, 10 Minutes Herbal Care text.\n\nDescription: A glossy green rectangular sachet with vivid branding and product imagery for an herbal hair colour shampoo."); + // Call our backend endpoint to parse PDF and query Gemini + const prompt = await api.extractPDF(pdfFile); + setExtractedPrompt(prompt); setCurrentStep(2); - } catch (err) { + } catch (err: any) { console.error(err); - alert("Failed to process PDF."); + alert(`Failed to process PDF: ${err.message || 'Unknown error'}`); } finally { setIsLoading(false); } @@ -80,22 +78,18 @@ export default function SparkPlug() { setIsLoading(false); return; } - - // Mocking the image generation with Nano Banana - await new Promise(r => setTimeout(r, 3000)); + // Call our backend endpoint which queries Gemini Imagen + const images = await api.generateImages(extractedPrompt); - // Placeholder images for the 4 views (front, back, left, right) - setGeneratedImages([ - "https://images.unsplash.com/photo-1629198688000-71f23e745b6e?w=400&q=80", // front - "https://images.unsplash.com/photo-1629198725848-18e55e975cc3?w=400&q=80", // back - "https://images.unsplash.com/photo-1629198725838-511bb7405234?w=400&q=80", // left - "https://images.unsplash.com/photo-1629198688000-71f23e745b6e?w=400&q=80", // right - ]); - - setCurrentStep(3); - } catch (err) { + if (images && images.length === 4) { + setGeneratedImages(images); + setCurrentStep(3); + } else { + throw new Error("Failed to generate exactly 4 images."); + } + } catch (err: any) { console.error(err); - alert("Failed to generate image."); + alert(`Failed to generate images: ${err.message || 'Unknown error'}`); } finally { setIsLoading(false); } @@ -116,17 +110,39 @@ export default function SparkPlug() { setIsLoading(false); return; } - - // Simulate Meshy generation - for (let i = 1; i <= 10; i++) { - await new Promise(r => setTimeout(r, 500)); - setModelProgress(i * 10); + // Call backend to start Meshy task + const taskId = await api.generate3D(generatedImages); + + // Poll Meshy status + let isDone = false; + let attempts = 0; + while (!isDone && attempts < 60) { // Max 5 mins (60 * 5s) + attempts++; + await new Promise(resolve => setTimeout(resolve, 5000)); + + const statusData = await api.get3DStatus(taskId); + const { status, progress, model_urls, task_error } = statusData; + + if (progress) { + setModelProgress(progress); + } + + if (status === 'SUCCEEDED') { + setGeneratedGlbUrl(model_urls.glb); + isDone = true; + setCurrentStep(4); + } else if (status === 'FAILED') { + throw new Error(`Meshy generation failed: ${task_error?.message || 'Unknown error'}`); + } } - setCurrentStep(4); - } catch (err) { + if (!isDone) { + throw new Error("Meshy task timed out after 5 minutes."); + } + + } catch (err: any) { console.error(err); - alert("Failed to generate 3D model."); + alert(`Failed to generate 3D model: ${err.message || 'Unknown error'}`); } finally { setIsLoading(false); } @@ -136,8 +152,9 @@ export default function SparkPlug() { setLoadingMessage('Saving project to your workspace...'); setIsLoading(true); try { - // Mock generated GLB URL from Meshy - const mockGlbUrl = "https://raw.githubusercontent.com/KhronosGroup/glTF-Sample-Models/master/2.0/BoxTextured/glTF-Binary/BoxTextured.glb"; + if (!generatedGlbUrl) { + throw new Error("No GLB URL available to save."); + } // Create project and save with model injected into scene_data const res = await api.saveProject({ @@ -152,7 +169,7 @@ export default function SparkPlug() { position: [0, 0, 0], rotation: [0, 0, 0], scale: [1, 1, 1], - url: mockGlbUrl + url: generatedGlbUrl } ] }