From 3fd8833457ddbf1045bba4ca8634d950f3ca87f7 Mon Sep 17 00:00:00 2001 From: Felix Kaspar Date: Fri, 20 Oct 2023 02:10:03 +0200 Subject: [PATCH] Stream Zip & Fixed Split in Node --- README.md | 2 +- functions/splitPDF.js | 1 - package-lock.json | 230 +++++++++++++++++++++++++++++++++++ package.json | 1 + public/traverseOperations.js | 1 + routes/api/index.js | 3 +- routes/api/workflow.js | 26 +++- traverseOperations.js | 32 ++++- 8 files changed, 284 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index d970e4c7..e4a5086a 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This is the development repository for the new StirlingPDF backend. With the pow ## Understanding Workflows -Workflows are the way you define what operations and their order should be applied to the PDF. +Workflows define how to apply operations to a PDF, including their order and relations with eachother. Workflows can be created via the web-ui and then exported or, if you want to brag a bit, you can create the JSON object yourself. diff --git a/functions/splitPDF.js b/functions/splitPDF.js index 01bb6636..ab68b4fe 100644 --- a/functions/splitPDF.js +++ b/functions/splitPDF.js @@ -20,6 +20,5 @@ export const splitPDF = async (snapshot, splitAfterPageArray) => { } subDocuments.push(await createSubDocument(pdfDoc, pagesArray)); pagesArray = []; - return subDocuments; }; \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 5260cde1..cee0c2a9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -39,11 +39,65 @@ "negotiator": "0.6.3" } }, + "archiver": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/archiver/-/archiver-6.0.1.tgz", + "integrity": "sha512-CXGy4poOLBKptiZH//VlWdFuUC1RESbdZjGjILwBuZ73P7WkAUN0htfSfBq/7k6FRFlpu7bg4JOkj1vU9G6jcQ==", + "requires": { + "archiver-utils": "^4.0.1", + "async": "^3.2.4", + "buffer-crc32": "^0.2.1", + "readable-stream": "^3.6.0", + "readdir-glob": "^1.1.2", + "tar-stream": "^3.0.0", + "zip-stream": "^5.0.1" + }, + "dependencies": { + "tar-stream": { + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.6.tgz", + "integrity": "sha512-B/UyjYwPpMBv+PaFSWAmtYjwdrlEaZQEhMIBFNC5oEG8lpiW8XjcSdmEaClj28ArfKScKHs2nshz3k2le6crsg==", + "requires": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + } + } + }, + "archiver-utils": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-4.0.1.tgz", + "integrity": "sha512-Q4Q99idbvzmgCTEAAhi32BkOyq8iVI5EwdO0PmBDSGIzzjYNdcFn7Q7k3OzbLy4kLUPXfJtG6fO2RjftXbobBg==", + "requires": { + "glob": "^8.0.0", + "graceful-fs": "^4.2.0", + "lazystream": "^1.0.0", + "lodash": "^4.17.15", + "normalize-path": "^3.0.0", + "readable-stream": "^3.6.0" + } + }, "array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" }, + "async": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", + "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" + }, + "b4a": { + "version": "1.6.4", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.4.tgz", + "integrity": "sha512-fpWrvyVHEKyeEvbKZTVOeZF3VSKKWtJxFIxX/jaVPf+cLbGUSitjb49pHLqPV2BUNNZ0LcoeEGfE/YCpyDYHIw==" + }, + "balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, "base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -78,6 +132,14 @@ "unpipe": "1.0.0" } }, + "brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "requires": { + "balanced-match": "^1.0.0" + } + }, "buffer": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", @@ -87,6 +149,11 @@ "ieee754": "^1.1.13" } }, + "buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==" + }, "busboy": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", @@ -109,6 +176,17 @@ "get-intrinsic": "^1.0.2" } }, + "compress-commons": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-5.0.1.tgz", + "integrity": "sha512-MPh//1cERdLtqwO3pOFLeXtpuai0Y2WCd5AhtKxznqM7WtaMYaOEMSgn45d9D10sIHSfIKE603HlOp8OPGrvag==", + "requires": { + "crc-32": "^1.2.0", + "crc32-stream": "^5.0.0", + "normalize-path": "^3.0.0", + "readable-stream": "^3.6.0" + } + }, "content-disposition": { "version": "0.5.4", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", @@ -132,6 +210,25 @@ "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==" }, + "core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" + }, + "crc-32": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", + "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==" + }, + "crc32-stream": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-5.0.0.tgz", + "integrity": "sha512-B0EPa1UK+qnpBZpG+7FgPCu0J2ETLpXq09o9BkLkEAhdB6Z61Qo4pJ3JYu0c+Qi+/SAL7QThqnzS06pmSSyZaw==", + "requires": { + "crc-32": "^1.2.0", + "readable-stream": "^3.4.0" + } + }, "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -229,6 +326,11 @@ "resolved": "https://registry.npmjs.org/fast-extend/-/fast-extend-1.0.2.tgz", "integrity": "sha512-XXA9RmlPatkFKUzqVZAFth18R4Wo+Xug/S+C7YlYA3xrXwfPlW3dqNwOb4hvQo7wZJ2cNDYhrYuPzVOfHy5/uQ==" }, + "fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==" + }, "finalhandler": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", @@ -263,6 +365,11 @@ "resolved": "https://registry.npmjs.org/fs-monkey/-/fs-monkey-0.3.3.tgz", "integrity": "sha512-FNUvuTAJ3CqCQb5ELn+qCbGR/Zllhf2HtwsdAtBi59s1WeCjKMT81fHcSu7dwIskqGVK+MmOrb7VOBlq3/SItw==" }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" + }, "function-bind": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", @@ -279,6 +386,23 @@ "has-symbols": "^1.0.3" } }, + "glob": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", + "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^5.0.1", + "once": "^1.3.0" + } + }, + "graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==" + }, "has": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/has/-/has-1.0.4.tgz", @@ -319,6 +443,15 @@ "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, "inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", @@ -329,6 +462,53 @@ "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==" }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + }, + "lazystream": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.1.tgz", + "integrity": "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==", + "requires": { + "readable-stream": "^2.0.5" + }, + "dependencies": { + "readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, + "lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, "media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -371,6 +551,14 @@ "mime-db": "1.52.0" } }, + "minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "requires": { + "brace-expansion": "^2.0.1" + } + }, "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -381,6 +569,11 @@ "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==" }, + "normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==" + }, "object-inspect": { "version": "1.13.0", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.0.tgz", @@ -428,6 +621,11 @@ "tslib": "^1.11.1" } }, + "process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, "proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -445,6 +643,11 @@ "side-channel": "^1.0.4" } }, + "queue-tick": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz", + "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==" + }, "range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", @@ -471,6 +674,14 @@ "util-deprecate": "^1.0.1" } }, + "readdir-glob": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/readdir-glob/-/readdir-glob-1.1.3.tgz", + "integrity": "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==", + "requires": { + "minimatch": "^5.1.0" + } + }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -544,6 +755,15 @@ "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==" }, + "streamx": { + "version": "2.15.1", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.15.1.tgz", + "integrity": "sha512-fQMzy2O/Q47rgwErk/eGeLu/roaFWV0jVsogDmrszM9uIw8L5OA+t+V93MgYlufNptfjmYR1tOMWhei/Eh7TQA==", + "requires": { + "fast-fifo": "^1.1.0", + "queue-tick": "^1.0.1" + } + }, "string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -607,6 +827,16 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + }, + "zip-stream": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-5.0.1.tgz", + "integrity": "sha512-UfZ0oa0C8LI58wJ+moL46BDIMgCQbnsb+2PoiJYtonhBsMh2bq1eRBVkvjfVsqbEHd9/EgKPUuL9saSSsec8OA==", + "requires": { + "archiver-utils": "^4.0.1", + "compress-commons": "^5.0.1", + "readable-stream": "^3.6.0" + } } } } diff --git a/package.json b/package.json index 35752ef9..0c554ac6 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "license": "ISC", "dependencies": { "@wasmer/wasmfs": "^0.12.0", + "archiver": "^6.0.1", "express": "^4.18.2", "express-fileupload": "^1.4.1", "pdf-lib": "^1.17.1" diff --git a/public/traverseOperations.js b/public/traverseOperations.js index a3c84041..8b8e1271 100644 --- a/public/traverseOperations.js +++ b/public/traverseOperations.js @@ -5,6 +5,7 @@ import { rotatePages } from "./functions/rotatePDF.js"; import { splitPDF } from "./functions/splitPDF.js"; import { organizeWaitOperations } from "./organizeWaitOperations.js"; +// TODO: Make this run with feedback like the server side func. export async function traverseOperations(operations, input) { const waitOperations = organizeWaitOperations(operations); const results = []; diff --git a/routes/api/index.js b/routes/api/index.js index 090fb61c..be18148c 100644 --- a/routes/api/index.js +++ b/routes/api/index.js @@ -6,7 +6,8 @@ const router = express.Router(); router.use(fileUpload()); router.get("/", function (req, res, next) { - res.status(501).json({"Error": "Unfinished Endpoint"}); + // TODO: Implement root api endpoint + res.status(501).json({"Error": "Unfinished Endpoint. This sould probably send some api docs?"}); }); router.use("/workflow", workflow); diff --git a/routes/api/workflow.js b/routes/api/workflow.js index f24ff015..91fdb83c 100644 --- a/routes/api/workflow.js +++ b/routes/api/workflow.js @@ -1,6 +1,7 @@ import express from 'express'; import crypto from 'crypto'; import stream from "stream"; +import Archiver from 'archiver'; import { traverseOperations } from "../../traverseOperations.js"; @@ -52,10 +53,10 @@ router.post("/:workflowUuid?", [ } console.log("Download"); - console.log(pdfResults); downloadHandler(res, pdfResults); } else { + console.log("Start Aync Workflow"); // TODO: UUID collision checks let workflowID = req.params.workflowUuid if(!workflowID) @@ -194,8 +195,27 @@ function downloadHandler(res, pdfResults) { res.status(500).json({"warning": "The workflow had no outputs."}); } else if(pdfResults.length > 1) { - res.status(501).json({"warning": "The workflow had multiple outputs, this is not implemented yet."}); - // TODO: Implement ZIP + // TODO: Also allow the user to download multiple files without zip compressen, because this is kind of slow... + res.writeHead(200, { + 'Content-Type': 'application/zip', + 'Content-disposition': 'attachment; filename=workflow-results.zip' + }); + + var zip = Archiver('zip'); + + // Stream the file to the user. + zip.pipe(res); + + console.log("Adding Files to ZIP..."); + + for (let i = 0; i < pdfResults.length; i++) { + // TODO: Implement other file types (mostly fro image & text extraction) + // TODO: Check for name collisions + zip.append(Buffer.from(pdfResults[i].buffer), { name: pdfResults[i].fileName + ".pdf" }); + } + + zip.finalize(); + console.log("Sent"); } else { const readStream = new stream.PassThrough(); diff --git a/traverseOperations.js b/traverseOperations.js index 818e8758..db69ff58 100644 --- a/traverseOperations.js +++ b/traverseOperations.js @@ -35,7 +35,21 @@ export async function * traverseOperations(operations, input) { break; case "wait": const waitOperation = waitOperations[operation.values.id]; - waitOperation.input.push(input); + + if(Array.isArray(input)) { + // waitOperation.input.concat(input); // May have unexpected concequences. Better throw an error for now. + throw new Error("Wait recieved an array as input. I don't know if this can happen, but if it does happen, I will investigate. Please share your workflow (:"); + } + else { + waitOperation.input.push(input); + } + + // Wait for all elements of previous split to finish + if(input.splitCount && input.splitCount > 0) { + input.splitCount--; + return; + } + waitOperation.waitCount--; if(waitOperation.waitCount == 0) { await nextOperation(waitOperation.doneOperation.operations, waitOperation.input); @@ -76,7 +90,6 @@ export async function * traverseOperations(operations, input) { } break; case "split": - // TODO: When a split goes into a wait function it might break the done condition, as it will count multiplpe times. if(Array.isArray(input)) { for (let i = 0; i < input.length; i++) { const splits = await splitPDF(input[i].buffer, operation.values["pagesToSplitAfterArray"]); @@ -84,9 +97,13 @@ export async function * traverseOperations(operations, input) { for (let j = 0; j < splits.length; j++) { const split = {}; split.originalFileName = input[i].originalFileName; - split.fileName = input[i].fileName + "_split"; + split.fileName = input[i].fileName + "_split" + j; split.buffer = splits[j]; - await nextOperation(operation.operations, split); + split.splitCount = splits.length * input.length; + // TODO: When a split goes into another split function and then into a wait function it might break the done condition, as it will count multiplpe times. + for await (const value of nextOperation(operation.operations, split)) { + yield value; + } } } } @@ -96,9 +113,12 @@ export async function * traverseOperations(operations, input) { for (let j = 0; j < splits.length; j++) { const split = {}; split.originalFileName = input.originalFileName; - split.fileName = input.fileName + "_split"; + split.fileName = input.fileName + "_split" + j; split.buffer = splits[j]; - await nextOperation(operation.operations, split); + split.splitCount = splits.length; + for await (const value of nextOperation(operation.operations, split)) { + yield value; + } } } break;