Refactor PDF extraction to use chunked upload to bypass proxy size limits

backend/server.js

@@ -323,16 +323,25 @@ const getKeysFromProfile = async (userId) => {
 };
 
 // 1. Extract PDF (Gemini)
-app.post('/api/sparkplug/extract', authenticate, upload.single('file'), async (req, res) => {
+app.post('/api/sparkplug/extract', authenticate, async (req, res) => {
-  if (!req.file) return res.status(400).json({ error: 'No file uploaded' });
+  const { fileUrl } = req.body;
+  if (!fileUrl) return res.status(400).json({ error: 'No fileUrl provided' });
   
   try {
     const keys = await getKeysFromProfile(req.user.id);
     const geminiKey = keys.GEMINI_API_KEY;
     if (!geminiKey) return res.status(400).json({ error: 'Gemini API key missing' });
 
-    // Parse PDF
+    // Read PDF from the local file system
-    const dataBuffer = fs.readFileSync(req.file.path);
+    // fileUrl is something like '/uploads/1788251962641-55866782-file.pdf'
+    const fileName = fileUrl.replace('/uploads/', '');
+    const localFilePath = path.join(uploadsDir, fileName);
+
+    if (!fs.existsSync(localFilePath)) {
+      return res.status(404).json({ error: 'File not found on server' });
+    }
+
+    const dataBuffer = fs.readFileSync(localFilePath);
     const base64Pdf = dataBuffer.toString('base64');
 
     // Call Gemini to extract prompt
@@ -364,8 +373,12 @@ app.post('/api/sparkplug/extract', authenticate, upload.single('file'), async (r
   } catch (err) {
     res.status(500).json({ error: err.message });
   } finally {
-    if (req.file) {
+    if (fileUrl) {
-      fs.unlinkSync(req.file.path); // cleanup uploaded PDF
+      const fileName = fileUrl.replace('/uploads/', '');
+      const localFilePath = path.join(uploadsDir, fileName);
+      if (fs.existsSync(localFilePath)) {
+        fs.unlinkSync(localFilePath); // cleanup uploaded PDF
+      }
     }
   }
 });

package-lock.json

@@ -1,11 +1,11 @@
 {
-  "name": "viz-3D_hun",
+  "name": "titan3d",
   "version": "0.0.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
-      "name": "viz-3D_hun",
+      "name": "titan3d",
       "version": "0.0.0",
       "dependencies": {
         "@fal-ai/client": "^1.10.1",
@@ -22,6 +22,7 @@
         "fflate": "^0.8.2",
         "framer-motion": "^12.38.0",
         "lucide-react": "^0.546.0",
+        "pdfjs-dist": "^6.3.289",
         "postprocessing": "^6.39.0",
         "react": "^19.0.0",
         "react-dom": "^19.0.0",
@@ -35,6 +36,7 @@
       "devDependencies": {
         "@types/express": "^4.17.21",
         "@types/node": "^22.14.0",
+        "@types/pdfjs-dist": "^2.10.377",
         "autoprefixer": "^10.4.21",
         "tailwindcss": "^4.1.14",
         "tsx": "^4.21.0",
@@ -845,6 +847,256 @@
         "node": ">= 18"
       }
     },
+    "node_modules/@napi-rs/canvas": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-1.0.8.tgz",
+      "integrity": "sha512-/SaLcvlqGWdm0HSCWMgHu7cjJiQXfP8/mOY+6dUyV9flQz7sPBBZ+ed2zYtoukojPmxOaL7bm+d/G4GeWWoN7g==",
+      "license": "MIT",
+      "optional": true,
+      "workspaces": [
+        "e2e/*"
+      ],
+      "engines": {
+        "node": ">= 10"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/Brooooooklyn"
+      },
+      "optionalDependencies": {
+        "@napi-rs/canvas-android-arm64": "1.0.8",
+        "@napi-rs/canvas-darwin-arm64": "1.0.8",
+        "@napi-rs/canvas-darwin-x64": "1.0.8",
+        "@napi-rs/canvas-linux-arm-gnueabihf": "1.0.8",
+        "@napi-rs/canvas-linux-arm64-gnu": "1.0.8",
+        "@napi-rs/canvas-linux-arm64-musl": "1.0.8",
+        "@napi-rs/canvas-linux-riscv64-gnu": "1.0.8",
+        "@napi-rs/canvas-linux-x64-gnu": "1.0.8",
+        "@napi-rs/canvas-linux-x64-musl": "1.0.8",
+        "@napi-rs/canvas-win32-arm64-msvc": "1.0.8",
+        "@napi-rs/canvas-win32-x64-msvc": "1.0.8"
+      }
+    },
+    "node_modules/@napi-rs/canvas-android-arm64": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-1.0.8.tgz",
+      "integrity": "sha512-5+nkh8i3gt6lqS/d2jTZ1xAn6tdgtB4Lf1mW6T0Qm5/rXNwBuV1sAEyLEWan5o9gJPU/GuvHR3rvSeZ+FaGrbw==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "android"
+      ],
+      "engines": {
+        "node": ">= 10"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/Brooooooklyn"
+      }
+    },
+    "node_modules/@napi-rs/canvas-darwin-arm64": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-1.0.8.tgz",
+      "integrity": "sha512-7jQ47gi+fZ7KJmfc/5rNyy1CYw/cu4kZ0KPIYbo9UUgSdW0bKQJpt+WihEor6s4Lyp7+xc3a+3HeyXmAEbbnPg==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": ">= 10"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/Brooooooklyn"
+      }
+    },
+    "node_modules/@napi-rs/canvas-darwin-x64": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-1.0.8.tgz",
+      "integrity": "sha512-rRjDMZs9pIRKGxgijwezplKc1RnJsqUokrA9h88bbTkqQ+7ePj0ZN4ZnZDy8Vu0tXs7KRlI2tQLaK4mx9QlxHg==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": ">= 10"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/Brooooooklyn"
+      }
+    },
+    "node_modules/@napi-rs/canvas-linux-arm-gnueabihf": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-1.0.8.tgz",
+      "integrity": "sha512-jGcCd+8ra6Q61xKqZeiItujTpp9a9eRLcQ0jW6qYNku+WpupqOPFPY0SrsuSnXFviJwkpKYT9p7QrB4lsf3LNQ==",
+      "cpu": [
+        "arm"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 10"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/Brooooooklyn"
+      }
+    },
+    "node_modules/@napi-rs/canvas-linux-arm64-gnu": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-1.0.8.tgz",
+      "integrity": "sha512-od6I2Y7kU7i1SwZYG2EKW8rWz6JiedtPpko4WEe1DDsiikrfaotVBCRaUTM5/yeZKaZ92EatoAS+5xG+6uJlYA==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 10"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/Brooooooklyn"
+      }
+    },
+    "node_modules/@napi-rs/canvas-linux-arm64-musl": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-1.0.8.tgz",
+      "integrity": "sha512-yYkPbJDJiWj6N0gASA3CAvRypZmVpJnxU0DQg3aBhneLDQde9TPLKADsQkobNoJUtTT/lj46aWpzT48PDb3Qcg==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 10"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/Brooooooklyn"
+      }
+    },
+    "node_modules/@napi-rs/canvas-linux-riscv64-gnu": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-1.0.8.tgz",
+      "integrity": "sha512-PB00MSKAp4VwK/xwe6duKxRKmH8UH4GIl1pqHSbxng0jnU9Dr7FwaDypDiqwNFZ774N+8G7mJLGuLtg9NTcQsg==",
+      "cpu": [
+        "riscv64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 10"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/Brooooooklyn"
+      }
+    },
+    "node_modules/@napi-rs/canvas-linux-x64-gnu": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-1.0.8.tgz",
+      "integrity": "sha512-TWM2XWJoitLiIPCvgJh7SriC+L/T9qkYCVzC66AidsZy0QP1hkKzBzVwshCdcA3q6fIn3yE0ISbq4lMJSy8jFw==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 10"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/Brooooooklyn"
+      }
+    },
+    "node_modules/@napi-rs/canvas-linux-x64-musl": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-1.0.8.tgz",
+      "integrity": "sha512-hb20MxKXXb5IB7AAwN8UHz9WRsa2HmdZfjsDCzjElwJoeV1aotVEwFU4FrFQcYQVzsJQLeaCc/2Qdt/0Q72mMg==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 10"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/Brooooooklyn"
+      }
+    },
+    "node_modules/@napi-rs/canvas-win32-arm64-msvc": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-arm64-msvc/-/canvas-win32-arm64-msvc-1.0.8.tgz",
+      "integrity": "sha512-WwPN08IXE4SkL+FhJyPz/iFnycMAUkbphFIT4cmKLlvbSU0Zfn1R7BGJ3Hqky1S89QUYc0Q4IOScXb/42Re9wQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">= 10"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/Brooooooklyn"
+      }
+    },
+    "node_modules/@napi-rs/canvas-win32-x64-msvc": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-1.0.8.tgz",
+      "integrity": "sha512-XkrVqKb+pxyba7kjy2LJvABFVBTE0DNpEl7MrG4OYUmaWarrXH+t54z/Czj2YxCKtizYTV4mg6phNm3x24qjhQ==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">= 10"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/Brooooooklyn"
+      }
+    },
     "node_modules/@protobufjs/aspromise": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
@@ -1746,6 +1998,16 @@
       "integrity": "sha512-ieXiYmgSRXUDeOntE1InxjWyvEelZGP63M+cGuquuRLuIKKT1osnkXjxev9B7d1nXSug5vpunx+gNlbVxMlC9A==",
       "license": "MIT"
     },
+    "node_modules/@types/pdfjs-dist": {
+      "version": "2.10.377",
+      "resolved": "https://registry.npmjs.org/@types/pdfjs-dist/-/pdfjs-dist-2.10.377.tgz",
+      "integrity": "sha512-RNBWpazVTkC5aiUIIo6d7ojlL0hJSoeQP1QCvTsVLW6qkNEpTALiI0DNGCYAGDfhbL0r17c/zNPQ15lk948UXg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "pdfjs-dist": "*"
+      }
+    },
     "node_modules/@types/qs": {
       "version": "6.15.0",
       "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.15.0.tgz",
@@ -3644,6 +3906,18 @@
       "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==",
       "license": "MIT"
     },
+    "node_modules/pdfjs-dist": {
+      "version": "6.3.289",
+      "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-6.3.289.tgz",
+      "integrity": "sha512-ZHjSVpDa3D6izMq8/04lvkhkATUmL9px6ChPaXc1k6nU2Mrhlg1/7F0bdUqCwUjw3NsPTfPZsMDUU6ZIcRaeQw==",
+      "license": "Apache-2.0",
+      "engines": {
+        "node": ">=22.13.0 || >=24"
+      },
+      "optionalDependencies": {
+        "@napi-rs/canvas": "^1.0.0"
+      }
+    },
     "node_modules/picocolors": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",

package.json

@@ -25,6 +25,7 @@
     "fflate": "^0.8.2",
     "framer-motion": "^12.38.0",
     "lucide-react": "^0.546.0",
+    "pdfjs-dist": "^6.3.289",
     "postprocessing": "^6.39.0",
     "react": "^19.0.0",
     "react-dom": "^19.0.0",
@@ -38,6 +39,7 @@
   "devDependencies": {
     "@types/express": "^4.17.21",
     "@types/node": "^22.14.0",
+    "@types/pdfjs-dist": "^2.10.377",
     "autoprefixer": "^10.4.21",
     "tailwindcss": "^4.1.14",
     "tsx": "^4.21.0",

src/lib/api.ts

@@ -199,15 +199,17 @@ export const api = {
   // ==========================================
   
   extractPDF: async (file: File) => {
-    const formData = new FormData();
+    // 1. First upload the file using the chunked uploader to bypass proxy size limits
-    formData.append('file', file);
+    const fileUrl = await api.uploadAsset(file);
     
+    // 2. Then ask the backend to extract text from the now-local file
     const res = await fetch(`${API_URL}/sparkplug/extract`, {
       method: 'POST',
       headers: {
+        'Content-Type': 'application/json',
         'Authorization': `Bearer ${localStorage.getItem('auth_token')}`
       },
-      body: formData
+      body: JSON.stringify({ fileUrl })
     });
     const data = await res.json();
     if (!res.ok) throw new Error(data.error);