From e941be02429928458c98184f0c5eab586e470ecf Mon Sep 17 00:00:00 2001 From: AI Bot Date: Wed, 2 Sep 2026 16:39:36 +0530 Subject: [PATCH] Refactor PDF extraction to use chunked upload to bypass proxy size limits --- backend/server.js | 25 ++++- package-lock.json | 278 +++++++++++++++++++++++++++++++++++++++++++++- package.json | 2 + src/lib/api.ts | 8 +- 4 files changed, 302 insertions(+), 11 deletions(-) diff --git a/backend/server.js b/backend/server.js index 129a28f..10087c2 100644 --- a/backend/server.js +++ b/backend/server.js @@ -323,16 +323,25 @@ const getKeysFromProfile = async (userId) => { }; // 1. Extract PDF (Gemini) -app.post('/api/sparkplug/extract', authenticate, upload.single('file'), async (req, res) => { - if (!req.file) return res.status(400).json({ error: 'No file uploaded' }); +app.post('/api/sparkplug/extract', authenticate, async (req, res) => { + const { fileUrl } = req.body; + if (!fileUrl) return res.status(400).json({ error: 'No fileUrl provided' }); try { const keys = await getKeysFromProfile(req.user.id); const geminiKey = keys.GEMINI_API_KEY; if (!geminiKey) return res.status(400).json({ error: 'Gemini API key missing' }); - // Parse PDF - const dataBuffer = fs.readFileSync(req.file.path); + // Read PDF from the local file system + // fileUrl is something like '/uploads/1788251962641-55866782-file.pdf' + const fileName = fileUrl.replace('/uploads/', ''); + const localFilePath = path.join(uploadsDir, fileName); + + if (!fs.existsSync(localFilePath)) { + return res.status(404).json({ error: 'File not found on server' }); + } + + const dataBuffer = fs.readFileSync(localFilePath); const base64Pdf = dataBuffer.toString('base64'); // Call Gemini to extract prompt @@ -364,8 +373,12 @@ app.post('/api/sparkplug/extract', authenticate, upload.single('file'), async (r } catch (err) { res.status(500).json({ error: err.message }); } finally { - if (req.file) { - fs.unlinkSync(req.file.path); // cleanup uploaded PDF + if (fileUrl) { + const fileName = fileUrl.replace('/uploads/', ''); + const localFilePath = path.join(uploadsDir, fileName); + if (fs.existsSync(localFilePath)) { + fs.unlinkSync(localFilePath); // cleanup uploaded PDF + } } } }); diff --git a/package-lock.json b/package-lock.json index 8a3446e..06915f6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,11 +1,11 @@ { - "name": "viz-3D_hun", + "name": "titan3d", "version": "0.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "viz-3D_hun", + "name": "titan3d", "version": "0.0.0", "dependencies": { "@fal-ai/client": "^1.10.1", @@ -22,6 +22,7 @@ "fflate": "^0.8.2", "framer-motion": "^12.38.0", "lucide-react": "^0.546.0", + "pdfjs-dist": "^6.3.289", "postprocessing": "^6.39.0", "react": "^19.0.0", "react-dom": "^19.0.0", @@ -35,6 +36,7 @@ "devDependencies": { "@types/express": "^4.17.21", "@types/node": "^22.14.0", + "@types/pdfjs-dist": "^2.10.377", "autoprefixer": "^10.4.21", "tailwindcss": "^4.1.14", "tsx": "^4.21.0", @@ -845,6 +847,256 @@ "node": ">= 18" } }, + "node_modules/@napi-rs/canvas": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-1.0.8.tgz", + "integrity": "sha512-/SaLcvlqGWdm0HSCWMgHu7cjJiQXfP8/mOY+6dUyV9flQz7sPBBZ+ed2zYtoukojPmxOaL7bm+d/G4GeWWoN7g==", + "license": "MIT", + "optional": true, + "workspaces": [ + "e2e/*" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "optionalDependencies": { + "@napi-rs/canvas-android-arm64": "1.0.8", + "@napi-rs/canvas-darwin-arm64": "1.0.8", + "@napi-rs/canvas-darwin-x64": "1.0.8", + "@napi-rs/canvas-linux-arm-gnueabihf": "1.0.8", + "@napi-rs/canvas-linux-arm64-gnu": "1.0.8", + "@napi-rs/canvas-linux-arm64-musl": "1.0.8", + "@napi-rs/canvas-linux-riscv64-gnu": "1.0.8", + "@napi-rs/canvas-linux-x64-gnu": "1.0.8", + "@napi-rs/canvas-linux-x64-musl": "1.0.8", + "@napi-rs/canvas-win32-arm64-msvc": "1.0.8", + "@napi-rs/canvas-win32-x64-msvc": "1.0.8" + } + }, + "node_modules/@napi-rs/canvas-android-arm64": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-1.0.8.tgz", + "integrity": "sha512-5+nkh8i3gt6lqS/d2jTZ1xAn6tdgtB4Lf1mW6T0Qm5/rXNwBuV1sAEyLEWan5o9gJPU/GuvHR3rvSeZ+FaGrbw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-darwin-arm64": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-1.0.8.tgz", + "integrity": "sha512-7jQ47gi+fZ7KJmfc/5rNyy1CYw/cu4kZ0KPIYbo9UUgSdW0bKQJpt+WihEor6s4Lyp7+xc3a+3HeyXmAEbbnPg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-darwin-x64": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-1.0.8.tgz", + "integrity": "sha512-rRjDMZs9pIRKGxgijwezplKc1RnJsqUokrA9h88bbTkqQ+7ePj0ZN4ZnZDy8Vu0tXs7KRlI2tQLaK4mx9QlxHg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm-gnueabihf": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-1.0.8.tgz", + "integrity": "sha512-jGcCd+8ra6Q61xKqZeiItujTpp9a9eRLcQ0jW6qYNku+WpupqOPFPY0SrsuSnXFviJwkpKYT9p7QrB4lsf3LNQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-gnu": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-1.0.8.tgz", + "integrity": "sha512-od6I2Y7kU7i1SwZYG2EKW8rWz6JiedtPpko4WEe1DDsiikrfaotVBCRaUTM5/yeZKaZ92EatoAS+5xG+6uJlYA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-musl": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-1.0.8.tgz", + "integrity": "sha512-yYkPbJDJiWj6N0gASA3CAvRypZmVpJnxU0DQg3aBhneLDQde9TPLKADsQkobNoJUtTT/lj46aWpzT48PDb3Qcg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-riscv64-gnu": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-1.0.8.tgz", + "integrity": "sha512-PB00MSKAp4VwK/xwe6duKxRKmH8UH4GIl1pqHSbxng0jnU9Dr7FwaDypDiqwNFZ774N+8G7mJLGuLtg9NTcQsg==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-gnu": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-1.0.8.tgz", + "integrity": "sha512-TWM2XWJoitLiIPCvgJh7SriC+L/T9qkYCVzC66AidsZy0QP1hkKzBzVwshCdcA3q6fIn3yE0ISbq4lMJSy8jFw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-musl": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-1.0.8.tgz", + "integrity": "sha512-hb20MxKXXb5IB7AAwN8UHz9WRsa2HmdZfjsDCzjElwJoeV1aotVEwFU4FrFQcYQVzsJQLeaCc/2Qdt/0Q72mMg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-win32-arm64-msvc": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-arm64-msvc/-/canvas-win32-arm64-msvc-1.0.8.tgz", + "integrity": "sha512-WwPN08IXE4SkL+FhJyPz/iFnycMAUkbphFIT4cmKLlvbSU0Zfn1R7BGJ3Hqky1S89QUYc0Q4IOScXb/42Re9wQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-win32-x64-msvc": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-1.0.8.tgz", + "integrity": "sha512-XkrVqKb+pxyba7kjy2LJvABFVBTE0DNpEl7MrG4OYUmaWarrXH+t54z/Czj2YxCKtizYTV4mg6phNm3x24qjhQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, "node_modules/@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -1746,6 +1998,16 @@ "integrity": "sha512-ieXiYmgSRXUDeOntE1InxjWyvEelZGP63M+cGuquuRLuIKKT1osnkXjxev9B7d1nXSug5vpunx+gNlbVxMlC9A==", "license": "MIT" }, + "node_modules/@types/pdfjs-dist": { + "version": "2.10.377", + "resolved": "https://registry.npmjs.org/@types/pdfjs-dist/-/pdfjs-dist-2.10.377.tgz", + "integrity": "sha512-RNBWpazVTkC5aiUIIo6d7ojlL0hJSoeQP1QCvTsVLW6qkNEpTALiI0DNGCYAGDfhbL0r17c/zNPQ15lk948UXg==", + "dev": true, + "license": "MIT", + "dependencies": { + "pdfjs-dist": "*" + } + }, "node_modules/@types/qs": { "version": "6.15.0", "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.15.0.tgz", @@ -3644,6 +3906,18 @@ "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==", "license": "MIT" }, + "node_modules/pdfjs-dist": { + "version": "6.3.289", + "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-6.3.289.tgz", + "integrity": "sha512-ZHjSVpDa3D6izMq8/04lvkhkATUmL9px6ChPaXc1k6nU2Mrhlg1/7F0bdUqCwUjw3NsPTfPZsMDUU6ZIcRaeQw==", + "license": "Apache-2.0", + "engines": { + "node": ">=22.13.0 || >=24" + }, + "optionalDependencies": { + "@napi-rs/canvas": "^1.0.0" + } + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", diff --git a/package.json b/package.json index 81d2548..342339a 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "fflate": "^0.8.2", "framer-motion": "^12.38.0", "lucide-react": "^0.546.0", + "pdfjs-dist": "^6.3.289", "postprocessing": "^6.39.0", "react": "^19.0.0", "react-dom": "^19.0.0", @@ -38,6 +39,7 @@ "devDependencies": { "@types/express": "^4.17.21", "@types/node": "^22.14.0", + "@types/pdfjs-dist": "^2.10.377", "autoprefixer": "^10.4.21", "tailwindcss": "^4.1.14", "tsx": "^4.21.0", diff --git a/src/lib/api.ts b/src/lib/api.ts index 45644a0..cf669d9 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -199,15 +199,17 @@ export const api = { // ========================================== extractPDF: async (file: File) => { - const formData = new FormData(); - formData.append('file', file); + // 1. First upload the file using the chunked uploader to bypass proxy size limits + const fileUrl = await api.uploadAsset(file); + // 2. Then ask the backend to extract text from the now-local file const res = await fetch(`${API_URL}/sparkplug/extract`, { method: 'POST', headers: { + 'Content-Type': 'application/json', 'Authorization': `Bearer ${localStorage.getItem('auth_token')}` }, - body: formData + body: JSON.stringify({ fileUrl }) }); const data = await res.json(); if (!res.ok) throw new Error(data.error);