mohanki/titan3d_react_node_sqllite
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor PDF extraction to use chunked upload to bypass proxy size limits
Build and Deploy / build-and-push (push) Successful in 30s
Details

Browse Source
This commit is contained in:
AI Bot
2026-09-02 16:39:36 +05:30
parent 7a45b7a262
commit e941be0242
4 changed files with 302 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/server.js
+19 -6
View File
@@ -323,16 +323,25 @@ const getKeysFromProfile = async (userId) => {
};
// 1. Extract PDF (Gemini)
app.post('/api/sparkplug/extract', authenticate, upload.single('file'), async (req, res) => {
if (!req.file) return res.status(400).json({ error: 'No file uploaded' });
app.post('/api/sparkplug/extract', authenticate, async (req, res) => {
const { fileUrl } = req.body;
if (!fileUrl) return res.status(400).json({ error: 'No fileUrl provided' });
try {
const keys = await getKeysFromProfile(req.user.id);
const geminiKey = keys.GEMINI_API_KEY;
if (!geminiKey) return res.status(400).json({ error: 'Gemini API key missing' });
// Parse PDF
const dataBuffer = fs.readFileSync(req.file.path);
// Read PDF from the local file system
// fileUrl is something like '/uploads/1788251962641-55866782-file.pdf'
const fileName = fileUrl.replace('/uploads/', '');
const localFilePath = path.join(uploadsDir, fileName);
if (!fs.existsSync(localFilePath)) {
return res.status(404).json({ error: 'File not found on server' });
}
const dataBuffer = fs.readFileSync(localFilePath);
const base64Pdf = dataBuffer.toString('base64');
// Call Gemini to extract prompt
@@ -364,8 +373,12 @@ app.post('/api/sparkplug/extract', authenticate, upload.single('file'), async (r
} catch (err) {
res.status(500).json({ error: err.message });
} finally {
if (req.file) {
fs.unlinkSync(req.file.path); // cleanup uploaded PDF
if (fileUrl) {
const fileName = fileUrl.replace('/uploads/', '');
const localFilePath = path.join(uploadsDir, fileName);
if (fs.existsSync(localFilePath)) {
fs.unlinkSync(localFilePath); // cleanup uploaded PDF
}
}
}
});
package-lock.json
Generated
+276 -2
View File
@@ -1,11 +1,11 @@
{
"name": "viz-3D_hun",
"name": "titan3d",
"version": "0.0.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "viz-3D_hun",
"name": "titan3d",
"version": "0.0.0",
"dependencies": {
"@fal-ai/client": "^1.10.1",
@@ -22,6 +22,7 @@
"fflate": "^0.8.2",
"framer-motion": "^12.38.0",
"lucide-react": "^0.546.0",
"pdfjs-dist": "^6.3.289",
"postprocessing": "^6.39.0",
"react": "^19.0.0",
"react-dom": "^19.0.0",
@@ -35,6 +36,7 @@
"devDependencies": {
"@types/express": "^4.17.21",
"@types/node": "^22.14.0",
"@types/pdfjs-dist": "^2.10.377",
"autoprefixer": "^10.4.21",
"tailwindcss": "^4.1.14",
"tsx": "^4.21.0",
@@ -845,6 +847,256 @@
"node": ">= 18"
}
},
"node_modules/@napi-rs/canvas": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-1.0.8.tgz",
"integrity": "sha512-/SaLcvlqGWdm0HSCWMgHu7cjJiQXfP8/mOY+6dUyV9flQz7sPBBZ+ed2zYtoukojPmxOaL7bm+d/G4GeWWoN7g==",
"license": "MIT",
"optional": true,
"workspaces": [
"e2e/*"
],
"engines": {
"node": ">= 10"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/Brooooooklyn"
},
"optionalDependencies": {
"@napi-rs/canvas-android-arm64": "1.0.8",
"@napi-rs/canvas-darwin-arm64": "1.0.8",
"@napi-rs/canvas-darwin-x64": "1.0.8",
"@napi-rs/canvas-linux-arm-gnueabihf": "1.0.8",
"@napi-rs/canvas-linux-arm64-gnu": "1.0.8",
"@napi-rs/canvas-linux-arm64-musl": "1.0.8",
"@napi-rs/canvas-linux-riscv64-gnu": "1.0.8",
"@napi-rs/canvas-linux-x64-gnu": "1.0.8",
"@napi-rs/canvas-linux-x64-musl": "1.0.8",
"@napi-rs/canvas-win32-arm64-msvc": "1.0.8",
"@napi-rs/canvas-win32-x64-msvc": "1.0.8"
}
},
"node_modules/@napi-rs/canvas-android-arm64": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-1.0.8.tgz",
"integrity": "sha512-5+nkh8i3gt6lqS/d2jTZ1xAn6tdgtB4Lf1mW6T0Qm5/rXNwBuV1sAEyLEWan5o9gJPU/GuvHR3rvSeZ+FaGrbw==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"android"
],
"engines": {
"node": ">= 10"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/Brooooooklyn"
}
},
"node_modules/@napi-rs/canvas-darwin-arm64": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-1.0.8.tgz",
"integrity": "sha512-7jQ47gi+fZ7KJmfc/5rNyy1CYw/cu4kZ0KPIYbo9UUgSdW0bKQJpt+WihEor6s4Lyp7+xc3a+3HeyXmAEbbnPg==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">= 10"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/Brooooooklyn"
}
},
"node_modules/@napi-rs/canvas-darwin-x64": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-1.0.8.tgz",
"integrity": "sha512-rRjDMZs9pIRKGxgijwezplKc1RnJsqUokrA9h88bbTkqQ+7ePj0ZN4ZnZDy8Vu0tXs7KRlI2tQLaK4mx9QlxHg==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">= 10"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/Brooooooklyn"
}
},
"node_modules/@napi-rs/canvas-linux-arm-gnueabihf": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-1.0.8.tgz",
"integrity": "sha512-jGcCd+8ra6Q61xKqZeiItujTpp9a9eRLcQ0jW6qYNku+WpupqOPFPY0SrsuSnXFviJwkpKYT9p7QrB4lsf3LNQ==",
"cpu": [
"arm"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">= 10"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/Brooooooklyn"
}
},
"node_modules/@napi-rs/canvas-linux-arm64-gnu": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-1.0.8.tgz",
"integrity": "sha512-od6I2Y7kU7i1SwZYG2EKW8rWz6JiedtPpko4WEe1DDsiikrfaotVBCRaUTM5/yeZKaZ92EatoAS+5xG+6uJlYA==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">= 10"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/Brooooooklyn"
}
},
"node_modules/@napi-rs/canvas-linux-arm64-musl": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-1.0.8.tgz",
"integrity": "sha512-yYkPbJDJiWj6N0gASA3CAvRypZmVpJnxU0DQg3aBhneLDQde9TPLKADsQkobNoJUtTT/lj46aWpzT48PDb3Qcg==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">= 10"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/Brooooooklyn"
}
},
"node_modules/@napi-rs/canvas-linux-riscv64-gnu": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-1.0.8.tgz",
"integrity": "sha512-PB00MSKAp4VwK/xwe6duKxRKmH8UH4GIl1pqHSbxng0jnU9Dr7FwaDypDiqwNFZ774N+8G7mJLGuLtg9NTcQsg==",
"cpu": [
"riscv64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">= 10"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/Brooooooklyn"
}
},
"node_modules/@napi-rs/canvas-linux-x64-gnu": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-1.0.8.tgz",
"integrity": "sha512-TWM2XWJoitLiIPCvgJh7SriC+L/T9qkYCVzC66AidsZy0QP1hkKzBzVwshCdcA3q6fIn3yE0ISbq4lMJSy8jFw==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">= 10"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/Brooooooklyn"
}
},
"node_modules/@napi-rs/canvas-linux-x64-musl": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-1.0.8.tgz",
"integrity": "sha512-hb20MxKXXb5IB7AAwN8UHz9WRsa2HmdZfjsDCzjElwJoeV1aotVEwFU4FrFQcYQVzsJQLeaCc/2Qdt/0Q72mMg==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">= 10"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/Brooooooklyn"
}
},
"node_modules/@napi-rs/canvas-win32-arm64-msvc": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-arm64-msvc/-/canvas-win32-arm64-msvc-1.0.8.tgz",
"integrity": "sha512-WwPN08IXE4SkL+FhJyPz/iFnycMAUkbphFIT4cmKLlvbSU0Zfn1R7BGJ3Hqky1S89QUYc0Q4IOScXb/42Re9wQ==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">= 10"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/Brooooooklyn"
}
},
"node_modules/@napi-rs/canvas-win32-x64-msvc": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-1.0.8.tgz",
"integrity": "sha512-XkrVqKb+pxyba7kjy2LJvABFVBTE0DNpEl7MrG4OYUmaWarrXH+t54z/Czj2YxCKtizYTV4mg6phNm3x24qjhQ==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">= 10"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/Brooooooklyn"
}
},
"node_modules/@protobufjs/aspromise": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
@@ -1746,6 +1998,16 @@
"integrity": "sha512-ieXiYmgSRXUDeOntE1InxjWyvEelZGP63M+cGuquuRLuIKKT1osnkXjxev9B7d1nXSug5vpunx+gNlbVxMlC9A==",
"license": "MIT"
},
"node_modules/@types/pdfjs-dist": {
"version": "2.10.377",
"resolved": "https://registry.npmjs.org/@types/pdfjs-dist/-/pdfjs-dist-2.10.377.tgz",
"integrity": "sha512-RNBWpazVTkC5aiUIIo6d7ojlL0hJSoeQP1QCvTsVLW6qkNEpTALiI0DNGCYAGDfhbL0r17c/zNPQ15lk948UXg==",
"dev": true,
"license": "MIT",
"dependencies": {
"pdfjs-dist": "*"
}
},
"node_modules/@types/qs": {
"version": "6.15.0",
"resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.15.0.tgz",
@@ -3644,6 +3906,18 @@
"integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==",
"license": "MIT"
},
"node_modules/pdfjs-dist": {
"version": "6.3.289",
"resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-6.3.289.tgz",
"integrity": "sha512-ZHjSVpDa3D6izMq8/04lvkhkATUmL9px6ChPaXc1k6nU2Mrhlg1/7F0bdUqCwUjw3NsPTfPZsMDUU6ZIcRaeQw==",
"license": "Apache-2.0",
"engines": {
"node": ">=22.13.0 || >=24"
},
"optionalDependencies": {
"@napi-rs/canvas": "^1.0.0"
}
},
"node_modules/picocolors": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",
package.json
+2
View File
@@ -25,6 +25,7 @@
"fflate": "^0.8.2",
"framer-motion": "^12.38.0",
"lucide-react": "^0.546.0",
"pdfjs-dist": "^6.3.289",
"postprocessing": "^6.39.0",
"react": "^19.0.0",
"react-dom": "^19.0.0",
@@ -38,6 +39,7 @@
"devDependencies": {
"@types/express": "^4.17.21",
"@types/node": "^22.14.0",
"@types/pdfjs-dist": "^2.10.377",
"autoprefixer": "^10.4.21",
"tailwindcss": "^4.1.14",
"tsx": "^4.21.0",
src/lib/api.ts
+5 -3
View File
@@ -199,15 +199,17 @@ export const api = {
// ==========================================
extractPDF: async (file: File) => {
const formData = new FormData();
formData.append('file', file);
// 1. First upload the file using the chunked uploader to bypass proxy size limits
const fileUrl = await api.uploadAsset(file);
// 2. Then ask the backend to extract text from the now-local file
const res = await fetch(`${API_URL}/sparkplug/extract`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${localStorage.getItem('auth_token')}`
},
body: formData
body: JSON.stringify({ fileUrl })
});
const data = await res.json();
if (!res.ok) throw new Error(data.error);