multi file api & documentation

README.md

@@ -2,6 +2,77 @@
 
 This is the development repository for the new StirlingPDF backend. With the power of JS, WASM & GO this will provide almost all functionality SPDF can do currently directly on the client. For automation purposes this will still provide an API to automate your workflows.
 
+## Try the new API!
+
+[![Run in Postman](https://run.pstmn.io/button.svg)](https://documenter.getpostman.com/view/30633786/2s9YRB1Wto)
+
+## Understanding Workflows
+
+Workflows are the way you define what operations and their order should be applied to the PDF.
+
+Workflows can be created via the web-ui and then exported or, if you want to brag a bit, you can create the JSON object yourself.
+
+### Basics
+
+To create your own, you have to understand a few key features first. You can also look at more examples our github repository.
+
+``` json
+{
+  "outputOptions": {
+    "zip": false
+  },
+  "operations": [
+    {
+      "type": "extract",
+      "values": {
+        "pagesToExtractArray": [0, 2]
+      },
+      "operations": []
+    }
+  ]
+}
+```
+
+The workflow above will extract the first (p\[0\]) and third (p\[2\]) page of the document.
+
+You can also nest workflows like this:
+
+``` json
+{
+  "outputOptions": {
+    "zip": false
+  },
+  "operations": [
+    {
+      "type": "extract",
+      "values": {
+        "pagesToExtractArray": [0, 2]
+      },
+      "operations": [
+        {
+          "type": "impose",
+          "values": {
+            "nup": 2, // 2 pages of the input docuemtn will be put on one page of the output document.
+            "format": "A4L" // A4L -> The page size of the Ouput will be an A4 in Landscape. You can also use other paper formats and "P" for portrait output. 
+          },
+          "operations": []
+        }
+      ]
+    }
+  ]
+}
+```
+
+If you look at it closely, you will see that the extract operation has another nested operation of the type impose. This workflow will produce a PDF with the 1st and 2nd page of the input on one single page.
+
+### Advanced
+
+If that is not enought for you usecase, there is also the possibility to connect operations with eachother.
+
+You can also do different operations to produce two different output PDFs from one input.
+
+If you are interested in learning about this, take a look at the Example workflows provided in the repository, ask on the discord, or wait for me to finish this documentation.
+
 ## Features
 
 ### New

package-lock.json

@@ -39,11 +39,6 @@
         "negotiator": "0.6.3"
       }
     },
-    "append-field": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz",
-      "integrity": "sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw=="
-    },
     "array-flatten": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
@@ -92,11 +87,6 @@
         "ieee754": "^1.1.13"
       }
     },
-    "buffer-from": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
-      "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="
-    },
     "busboy": {
       "version": "1.6.0",
       "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz",
@@ -119,46 +109,6 @@
         "get-intrinsic": "^1.0.2"
       }
     },
-    "concat-stream": {
-      "version": "1.6.2",
-      "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
-      "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
-      "requires": {
-        "buffer-from": "^1.0.0",
-        "inherits": "^2.0.3",
-        "readable-stream": "^2.2.2",
-        "typedarray": "^0.0.6"
-      },
-      "dependencies": {
-        "readable-stream": {
-          "version": "2.3.8",
-          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
-          "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
-          "requires": {
-            "core-util-is": "~1.0.0",
-            "inherits": "~2.0.3",
-            "isarray": "~1.0.0",
-            "process-nextick-args": "~2.0.0",
-            "safe-buffer": "~5.1.1",
-            "string_decoder": "~1.1.1",
-            "util-deprecate": "~1.0.1"
-          }
-        },
-        "safe-buffer": {
-          "version": "5.1.2",
-          "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
-          "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
-        },
-        "string_decoder": {
-          "version": "1.1.1",
-          "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
-          "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
-          "requires": {
-            "safe-buffer": "~5.1.0"
-          }
-        }
-      }
-    },
     "content-disposition": {
       "version": "0.5.4",
       "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
@@ -182,11 +132,6 @@
       "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
       "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ=="
     },
-    "core-util-is": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz",
-      "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ=="
-    },
     "debug": {
       "version": "2.6.9",
       "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
@@ -271,6 +216,14 @@
         "vary": "~1.1.2"
       }
     },
+    "express-fileupload": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/express-fileupload/-/express-fileupload-1.4.1.tgz",
+      "integrity": "sha512-9F6SkbxbEOA9cYOBZ8tnn238jL+bGfacQuUO/JqPWp5t+piUcoDcESvKwAXsQV7IHGxmI5bMj3QxMWOKOIsMCg==",
+      "requires": {
+        "busboy": "^1.6.0"
+      }
+    },
     "fast-extend": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/fast-extend/-/fast-extend-1.0.2.tgz",
@@ -376,11 +329,6 @@
       "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
       "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="
     },
-    "isarray": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
-      "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="
-    },
     "media-typer": {
       "version": "0.3.0",
       "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
@@ -423,48 +371,16 @@
         "mime-db": "1.52.0"
       }
     },
-    "minimist": {
-      "version": "1.2.8",
-      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz",
-      "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA=="
-    },
-    "mkdirp": {
-      "version": "0.5.6",
-      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz",
-      "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==",
-      "requires": {
-        "minimist": "^1.2.6"
-      }
-    },
     "ms": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
       "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
     },
-    "multer": {
-      "version": "1.4.5-lts.1",
-      "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.5-lts.1.tgz",
-      "integrity": "sha512-ywPWvcDMeH+z9gQq5qYHCCy+ethsk4goepZ45GLD63fOu0YcNecQxi64nDs3qluZB+murG3/D4dJ7+dGctcCQQ==",
-      "requires": {
-        "append-field": "^1.0.0",
-        "busboy": "^1.0.0",
-        "concat-stream": "^1.5.2",
-        "mkdirp": "^0.5.4",
-        "object-assign": "^4.1.1",
-        "type-is": "^1.6.4",
-        "xtend": "^4.0.0"
-      }
-    },
     "negotiator": {
       "version": "0.6.3",
       "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
       "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg=="
     },
-    "object-assign": {
-      "version": "4.1.1",
-      "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
-      "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="
-    },
     "object-inspect": {
       "version": "1.13.0",
       "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.0.tgz",
@@ -512,11 +428,6 @@
         "tslib": "^1.11.1"
       }
     },
-    "process-nextick-args": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
-      "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
-    },
     "proxy-addr": {
       "version": "2.0.7",
       "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz",
@@ -672,11 +583,6 @@
         "mime-types": "~2.1.24"
       }
     },
-    "typedarray": {
-      "version": "0.0.6",
-      "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
-      "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA=="
-    },
     "unpipe": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
@@ -701,11 +607,6 @@
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
       "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="
-    },
-    "xtend": {
-      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
-      "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="
     }
   }
 }

package.json

@@ -11,7 +11,7 @@
   "dependencies": {
     "@wasmer/wasmfs": "^0.12.0",
     "express": "^4.18.2",
-    "multer": "^1.4.5-lts.1",
+    "express-fileupload": "^1.4.1",
     "pdf-lib": "^1.17.1"
   },
   "type": "module"

routes/api/index.js

@@ -1,7 +1,9 @@
 import express from 'express';
 import workflow from './workflow.js';
+import fileUpload from 'express-fileupload';
 
 const router = express.Router();
+router.use(fileUpload());
 
 router.get("/", function (req, res, next) {
     res.status(501).json({"Error": "Unfinished Endpoint"});

routes/api/workflow.js

@@ -1,5 +1,4 @@
 import express from 'express';
-import multer from 'multer';
 import crypto from 'crypto';
 import stream from "stream";
 
@@ -10,19 +9,27 @@ const activeWorkflows = {};
 const router = express.Router();
 
 router.post("/:workflowUuid?", [
-    multer().array("files"),
+    async (req, res) => {
-    async (req, res, next) => {
+        if(req.files == null) {
-        const workflow = JSON.parse(req.body.workflow);
+            res.status(400).json({"error": "No files were uploaded."});
-        console.log("fileCount: ", req.files.length);
+            return;
-        console.log("workflow: ", workflow);
+        }
 
-        // TODO: Validate
+        if(Array.isArray(req.files.files)) {
+            req.files = req.files.files;
+        }
+        else {
+            req.files = [req.files.files];
+        }
+
+        const workflow = JSON.parse(req.body.workflow);
+        // TODO: Validate input further (json may fail or not be a valid workflow)
 
         const inputs = await Promise.all(req.files.map(async file => {
             return {
-                originalFileName: file.originalname.replace(/\.[^/.]+$/, ""),
+                originalFileName: file.name.replace(/\.[^/.]+$/, ""),
-                fileName: file.originalname.replace(/\.[^/.]+$/, ""),
+                fileName: file.name.replace(/\.[^/.]+$/, ""),
-                buffer: new Uint8Array(await file.buffer)
+                buffer: new Uint8Array(await file.data)
             }
         }));
 
@@ -37,11 +44,15 @@ router.post("/:workflowUuid?", [
             while (true) {
                 iteration = await traverse.next();
                 if (iteration.done) {
+                    console.log(iteration.value);
                     pdfResults = iteration.value;
+                    console.log("Done");
                     break;
                 }
             }
 
+            console.log("Download");
+            console.log(pdfResults);
             downloadHandler(res, pdfResults);
         }
         else {
@@ -59,8 +70,7 @@ router.post("/:workflowUuid?", [
             }
             const activeWorkflow = activeWorkflows[workflowID];
 
-            res.status(501).json({
+            res.status(200).json({
-                "warning": "Unfinished Endpoint",
                 "workflowID": workflowID,
                 "data-recieved": {
                     "fileCount": req.files.length,
@@ -77,7 +87,7 @@ router.post("/:workflowUuid?", [
                 if (iteration.done) {
                     pdfResults = iteration.value;
                     if(activeWorkflow.eventStream) {
-                        activeWorkflow.eventStream.write(`data: processing done`);
+                        activeWorkflow.eventStream.write(`data: processing done\n\n`);
                         activeWorkflow.eventStream.end();
                     }
                     break;
@@ -93,7 +103,14 @@ router.post("/:workflowUuid?", [
 ]);
 
 router.get("/progress/:workflowUuid", (req, res, nex) => {
-    // TODO: Validation
+    if(!req.params.workflowUuid) {
+        res.status(400).json({"error": "No workflowUuid weres provided."});
+        return;
+    }
+    if(!activeWorkflows.hasOwnProperty(req.params.workflowUuid)) {
+        res.status(400).json({"error": `No workflow with workflowUuid "${req.params.workflowUuid}" was found.`});
+        return;
+    }
 
     // Return current progress
     const workflow = activeWorkflows[req.params.workflowUuid];
@@ -101,7 +118,16 @@ router.get("/progress/:workflowUuid", (req, res, nex) => {
 });
 
 router.get("/progress-stream/:workflowUuid", (req, res, nex) => {
-    // TODO: Validation
+    if(!req.params.workflowUuid) {
+        res.status(400).json({"error": "No workflowUuid weres provided."});
+        return;
+    }
+    if(!activeWorkflows.hasOwnProperty(req.params.workflowUuid)) {
+        res.status(400).json({"error": `No workflow with workflowUuid "${req.params.workflowUuid}" was found.`});
+        return;
+    }
+
+    // TODO: Check if already done
 
     // Send realtime updates
     res.setHeader('Cache-Control', 'no-cache');
@@ -120,7 +146,14 @@ router.get("/progress-stream/:workflowUuid", (req, res, nex) => {
 });
 
 router.get("/result/:workflowUuid", (req, res, nex) => {
-    // TODO: Validation
+    if(!req.params.workflowUuid) {
+        res.status(400).json({"error": "No workflowUuid weres provided."});
+        return;
+    }
+    if(!activeWorkflows.hasOwnProperty(req.params.workflowUuid)) {
+        res.status(400).json({"error": `No workflow with workflowUuid "${req.params.workflowUuid}" was found.`});
+        return;
+    }
 
     /* 
      * If workflow isn't done return error
@@ -139,6 +172,15 @@ router.get("/result/:workflowUuid", (req, res, nex) => {
 });
 
 router.post("/abort/:workflowUuid", (req, res, nex) => {
+    if(!req.params.workflowUuid) {
+        res.status(400).json({"error": "No workflowUuid weres provided."});
+        return;
+    }
+    if(!activeWorkflows.hasOwnProperty(req.params.workflowUuid)) {
+        res.status(400).json({"error": `No workflow with workflowUuid "${req.params.workflowUuid}" was found.`});
+        return;
+    }
+
     // TODO: Abort workflow
     res.status(501).json({"warning": "Abortion has not been implemented yet."});
 });
@@ -148,7 +190,10 @@ function generateWorkflowID() {
 }
 
 function downloadHandler(res, pdfResults) {
-    if(pdfResults.length > 1) {
+    if(pdfResults.length == 0) {
+        res.status(500).json({"warning": "The workflow had no outputs."});
+    } 
+    else if(pdfResults.length > 1) {
         res.status(501).json({"warning": "The workflow had multiple outputs, this is not implemented yet."});
         // TODO: Implement ZIP
     }