From 8193d5044a2e101cd57a63dc175015041e6ce74a Mon Sep 17 00:00:00 2001 From: Felix Kaspar Date: Fri, 20 Oct 2023 00:10:03 +0200 Subject: [PATCH] multi file api & documentation --- README.md | 71 +++++++++++++++++++++++++ package-lock.json | 115 +++-------------------------------------- package.json | 2 +- routes/api/index.js | 2 + routes/api/workflow.js | 79 ++++++++++++++++++++++------ 5 files changed, 144 insertions(+), 125 deletions(-) diff --git a/README.md b/README.md index 5c2823ab..d970e4c7 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,77 @@ This is the development repository for the new StirlingPDF backend. With the power of JS, WASM & GO this will provide almost all functionality SPDF can do currently directly on the client. For automation purposes this will still provide an API to automate your workflows. +## Try the new API! + +[![Run in Postman](https://run.pstmn.io/button.svg)](https://documenter.getpostman.com/view/30633786/2s9YRB1Wto) + +## Understanding Workflows + +Workflows are the way you define what operations and their order should be applied to the PDF. + +Workflows can be created via the web-ui and then exported or, if you want to brag a bit, you can create the JSON object yourself. + +### Basics + +To create your own, you have to understand a few key features first. You can also look at more examples our github repository. + +``` json +{ + "outputOptions": { + "zip": false + }, + "operations": [ + { + "type": "extract", + "values": { + "pagesToExtractArray": [0, 2] + }, + "operations": [] + } + ] +} +``` + +The workflow above will extract the first (p\[0\]) and third (p\[2\]) page of the document. + +You can also nest workflows like this: + +``` json +{ + "outputOptions": { + "zip": false + }, + "operations": [ + { + "type": "extract", + "values": { + "pagesToExtractArray": [0, 2] + }, + "operations": [ + { + "type": "impose", + "values": { + "nup": 2, // 2 pages of the input docuemtn will be put on one page of the output document. + "format": "A4L" // A4L -> The page size of the Ouput will be an A4 in Landscape. You can also use other paper formats and "P" for portrait output. + }, + "operations": [] + } + ] + } + ] +} +``` + +If you look at it closely, you will see that the extract operation has another nested operation of the type impose. This workflow will produce a PDF with the 1st and 2nd page of the input on one single page. + +### Advanced + +If that is not enought for you usecase, there is also the possibility to connect operations with eachother. + +You can also do different operations to produce two different output PDFs from one input. + +If you are interested in learning about this, take a look at the Example workflows provided in the repository, ask on the discord, or wait for me to finish this documentation. + ## Features ### New diff --git a/package-lock.json b/package-lock.json index c408197e..5260cde1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -39,11 +39,6 @@ "negotiator": "0.6.3" } }, - "append-field": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz", - "integrity": "sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==" - }, "array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -92,11 +87,6 @@ "ieee754": "^1.1.13" } }, - "buffer-from": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", - "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" - }, "busboy": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", @@ -119,46 +109,6 @@ "get-intrinsic": "^1.0.2" } }, - "concat-stream": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", - "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", - "requires": { - "buffer-from": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^2.2.2", - "typedarray": "^0.0.6" - }, - "dependencies": { - "readable-stream": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", - "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } - } - } - }, "content-disposition": { "version": "0.5.4", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", @@ -182,11 +132,6 @@ "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==" }, - "core-util-is": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", - "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" - }, "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -271,6 +216,14 @@ "vary": "~1.1.2" } }, + "express-fileupload": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/express-fileupload/-/express-fileupload-1.4.1.tgz", + "integrity": "sha512-9F6SkbxbEOA9cYOBZ8tnn238jL+bGfacQuUO/JqPWp5t+piUcoDcESvKwAXsQV7IHGxmI5bMj3QxMWOKOIsMCg==", + "requires": { + "busboy": "^1.6.0" + } + }, "fast-extend": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/fast-extend/-/fast-extend-1.0.2.tgz", @@ -376,11 +329,6 @@ "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==" }, - "isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" - }, "media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -423,48 +371,16 @@ "mime-db": "1.52.0" } }, - "minimist": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", - "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==" - }, - "mkdirp": { - "version": "0.5.6", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", - "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", - "requires": { - "minimist": "^1.2.6" - } - }, "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, - "multer": { - "version": "1.4.5-lts.1", - "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.5-lts.1.tgz", - "integrity": "sha512-ywPWvcDMeH+z9gQq5qYHCCy+ethsk4goepZ45GLD63fOu0YcNecQxi64nDs3qluZB+murG3/D4dJ7+dGctcCQQ==", - "requires": { - "append-field": "^1.0.0", - "busboy": "^1.0.0", - "concat-stream": "^1.5.2", - "mkdirp": "^0.5.4", - "object-assign": "^4.1.1", - "type-is": "^1.6.4", - "xtend": "^4.0.0" - } - }, "negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==" }, - "object-assign": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", - "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==" - }, "object-inspect": { "version": "1.13.0", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.0.tgz", @@ -512,11 +428,6 @@ "tslib": "^1.11.1" } }, - "process-nextick-args": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" - }, "proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -672,11 +583,6 @@ "mime-types": "~2.1.24" } }, - "typedarray": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", - "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==" - }, "unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", @@ -701,11 +607,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" - }, - "xtend": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" } } } diff --git a/package.json b/package.json index 7d29d67c..35752ef9 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "dependencies": { "@wasmer/wasmfs": "^0.12.0", "express": "^4.18.2", - "multer": "^1.4.5-lts.1", + "express-fileupload": "^1.4.1", "pdf-lib": "^1.17.1" }, "type": "module" diff --git a/routes/api/index.js b/routes/api/index.js index b75d742d..090fb61c 100644 --- a/routes/api/index.js +++ b/routes/api/index.js @@ -1,7 +1,9 @@ import express from 'express'; import workflow from './workflow.js'; +import fileUpload from 'express-fileupload'; const router = express.Router(); +router.use(fileUpload()); router.get("/", function (req, res, next) { res.status(501).json({"Error": "Unfinished Endpoint"}); diff --git a/routes/api/workflow.js b/routes/api/workflow.js index cef23375..f24ff015 100644 --- a/routes/api/workflow.js +++ b/routes/api/workflow.js @@ -1,5 +1,4 @@ import express from 'express'; -import multer from 'multer'; import crypto from 'crypto'; import stream from "stream"; @@ -10,19 +9,27 @@ const activeWorkflows = {}; const router = express.Router(); router.post("/:workflowUuid?", [ - multer().array("files"), - async (req, res, next) => { - const workflow = JSON.parse(req.body.workflow); - console.log("fileCount: ", req.files.length); - console.log("workflow: ", workflow); + async (req, res) => { + if(req.files == null) { + res.status(400).json({"error": "No files were uploaded."}); + return; + } - // TODO: Validate + if(Array.isArray(req.files.files)) { + req.files = req.files.files; + } + else { + req.files = [req.files.files]; + } + + const workflow = JSON.parse(req.body.workflow); + // TODO: Validate input further (json may fail or not be a valid workflow) const inputs = await Promise.all(req.files.map(async file => { return { - originalFileName: file.originalname.replace(/\.[^/.]+$/, ""), - fileName: file.originalname.replace(/\.[^/.]+$/, ""), - buffer: new Uint8Array(await file.buffer) + originalFileName: file.name.replace(/\.[^/.]+$/, ""), + fileName: file.name.replace(/\.[^/.]+$/, ""), + buffer: new Uint8Array(await file.data) } })); @@ -37,11 +44,15 @@ router.post("/:workflowUuid?", [ while (true) { iteration = await traverse.next(); if (iteration.done) { + console.log(iteration.value); pdfResults = iteration.value; + console.log("Done"); break; } } + console.log("Download"); + console.log(pdfResults); downloadHandler(res, pdfResults); } else { @@ -59,8 +70,7 @@ router.post("/:workflowUuid?", [ } const activeWorkflow = activeWorkflows[workflowID]; - res.status(501).json({ - "warning": "Unfinished Endpoint", + res.status(200).json({ "workflowID": workflowID, "data-recieved": { "fileCount": req.files.length, @@ -77,7 +87,7 @@ router.post("/:workflowUuid?", [ if (iteration.done) { pdfResults = iteration.value; if(activeWorkflow.eventStream) { - activeWorkflow.eventStream.write(`data: processing done`); + activeWorkflow.eventStream.write(`data: processing done\n\n`); activeWorkflow.eventStream.end(); } break; @@ -93,7 +103,14 @@ router.post("/:workflowUuid?", [ ]); router.get("/progress/:workflowUuid", (req, res, nex) => { - // TODO: Validation + if(!req.params.workflowUuid) { + res.status(400).json({"error": "No workflowUuid weres provided."}); + return; + } + if(!activeWorkflows.hasOwnProperty(req.params.workflowUuid)) { + res.status(400).json({"error": `No workflow with workflowUuid "${req.params.workflowUuid}" was found.`}); + return; + } // Return current progress const workflow = activeWorkflows[req.params.workflowUuid]; @@ -101,7 +118,16 @@ router.get("/progress/:workflowUuid", (req, res, nex) => { }); router.get("/progress-stream/:workflowUuid", (req, res, nex) => { - // TODO: Validation + if(!req.params.workflowUuid) { + res.status(400).json({"error": "No workflowUuid weres provided."}); + return; + } + if(!activeWorkflows.hasOwnProperty(req.params.workflowUuid)) { + res.status(400).json({"error": `No workflow with workflowUuid "${req.params.workflowUuid}" was found.`}); + return; + } + + // TODO: Check if already done // Send realtime updates res.setHeader('Cache-Control', 'no-cache'); @@ -120,7 +146,14 @@ router.get("/progress-stream/:workflowUuid", (req, res, nex) => { }); router.get("/result/:workflowUuid", (req, res, nex) => { - // TODO: Validation + if(!req.params.workflowUuid) { + res.status(400).json({"error": "No workflowUuid weres provided."}); + return; + } + if(!activeWorkflows.hasOwnProperty(req.params.workflowUuid)) { + res.status(400).json({"error": `No workflow with workflowUuid "${req.params.workflowUuid}" was found.`}); + return; + } /* * If workflow isn't done return error @@ -139,6 +172,15 @@ router.get("/result/:workflowUuid", (req, res, nex) => { }); router.post("/abort/:workflowUuid", (req, res, nex) => { + if(!req.params.workflowUuid) { + res.status(400).json({"error": "No workflowUuid weres provided."}); + return; + } + if(!activeWorkflows.hasOwnProperty(req.params.workflowUuid)) { + res.status(400).json({"error": `No workflow with workflowUuid "${req.params.workflowUuid}" was found.`}); + return; + } + // TODO: Abort workflow res.status(501).json({"warning": "Abortion has not been implemented yet."}); }); @@ -148,7 +190,10 @@ function generateWorkflowID() { } function downloadHandler(res, pdfResults) { - if(pdfResults.length > 1) { + if(pdfResults.length == 0) { + res.status(500).json({"warning": "The workflow had no outputs."}); + } + else if(pdfResults.length > 1) { res.status(501).json({"warning": "The workflow had multiple outputs, this is not implemented yet."}); // TODO: Implement ZIP }