From f6b44792009478284aa255c3783b04293c2972d9 Mon Sep 17 00:00:00 2001 From: Felix Kaspar Date: Sun, 22 Oct 2023 01:11:17 +0200 Subject: [PATCH] implemented new shared-traverser on browser --- functions/index.js => functions.js | 16 +- public/functions.js | 38 ++++ public/index.js | 20 +- public/traverseOperations.js | 278 ++++++++++---------------- public/wasm/pdfcpu-wrapper-browser.js | 2 +- routes/api/workflow.js | 5 +- traverseOperations.js | 134 ------------- 7 files changed, 169 insertions(+), 324 deletions(-) rename functions/index.js => functions.js (57%) create mode 100644 public/functions.js delete mode 100644 traverseOperations.js diff --git a/functions/index.js b/functions.js similarity index 57% rename from functions/index.js rename to functions.js index 8960f958..c6260f4a 100644 --- a/functions/index.js +++ b/functions.js @@ -1,13 +1,13 @@ import PDFLib from 'pdf-lib'; -import * as pdfcpuWraopper from "../public/wasm/pdfcpu-wrapper-node.js"; +import * as pdfcpuWraopper from "./public/wasm/pdfcpu-wrapper-node.js"; -import { extractPages as dependantExtractPages } from "../public/functions/extractPages.js"; -import { impose as dependantImpose } from '../public/functions/impose.js'; -import { mergePDFs as dependantMergePDFs } from '../public/functions/mergePDFs.js'; -import { rotatePages as dependantRotatePages } from '../public/functions/rotatePages.js'; -import { scaleContent as dependantScaleContent} from '../public/functions/scaleContent.js'; -import { scalePage as dependantScalePage } from '../public/functions/scalePage.js'; -import { splitPDF as dependantSplitPDF } from '../public/functions/splitPDF.js'; +import { extractPages as dependantExtractPages } from "./public/functions/extractPages.js"; +import { impose as dependantImpose } from './public/functions/impose.js'; +import { mergePDFs as dependantMergePDFs } from './public/functions/mergePDFs.js'; +import { rotatePages as dependantRotatePages } from './public/functions/rotatePages.js'; +import { scaleContent as dependantScaleContent} from './public/functions/scaleContent.js'; +import { scalePage as dependantScalePage } from './public/functions/scalePage.js'; +import { splitPDF as dependantSplitPDF } from './public/functions/splitPDF.js'; export async function extractPages(snapshot, pagesToExtractArray) { return dependantExtractPages(snapshot, pagesToExtractArray, PDFLib); diff --git a/public/functions.js b/public/functions.js new file mode 100644 index 00000000..ad7ae00e --- /dev/null +++ b/public/functions.js @@ -0,0 +1,38 @@ +// PDFLib gets importet via index.html script-tag +import * as pdfcpuWraopper from "./wasm/pdfcpu-wrapper-browser.js"; + +import { extractPages as dependantExtractPages } from "./functions/extractPages.js"; +import { impose as dependantImpose } from './functions/impose.js'; +import { mergePDFs as dependantMergePDFs } from './functions/mergePDFs.js'; +import { rotatePages as dependantRotatePages } from './functions/rotatePages.js'; +import { scaleContent as dependantScaleContent} from './functions/scaleContent.js'; +import { scalePage as dependantScalePage } from './functions/scalePage.js'; +import { splitPDF as dependantSplitPDF } from './functions/splitPDF.js'; + +export async function extractPages(snapshot, pagesToExtractArray) { + return dependantExtractPages(snapshot, pagesToExtractArray, PDFLib); +} + +export async function impose(snapshot, nup, format) { + return dependantImpose(snapshot, nup, format, pdfcpuWraopper); +} + +export async function mergePDFs(snapshots) { + return dependantMergePDFs(snapshots, PDFLib); +} + +export async function rotatePages(snapshot, rotation) { + return dependantRotatePages(snapshot, rotation, PDFLib); +} + +export async function scaleContent(snapshot, scaleFactor) { + return dependantScaleContent(snapshot, scaleFactor, PDFLib); +} + +export async function scalePage(snapshot, pageSize) { + return dependantScalePage(snapshot, pageSize, PDFLib); +} + +export async function splitPDF(snapshot, splitAfterPageArray) { + return dependantSplitPDF(snapshot, splitAfterPageArray, PDFLib); +} \ No newline at end of file diff --git a/public/index.js b/public/index.js index 7153dab2..55709d10 100644 --- a/public/index.js +++ b/public/index.js @@ -2,12 +2,15 @@ import { scaleContent } from "./functions/scaleContent.js"; import { scalePage, PageSize } from "./functions/scalePage.js"; import * as exampleWorkflows from "./exampleWorkflows.js"; import { traverseOperations } from "./traverseOperations.js"; +import * as Functions from "./functions.js"; (async (workflow) => { const pdfFileInput = document.getElementById('pdfFile'); const doneButton = document.getElementById("doneButton"); doneButton.addEventListener('click', async (e) => { + console.log("Starting..."); + const files = Array.from(pdfFileInput.files); console.log(files); const inputs = await Promise.all(files.map(async file => { @@ -19,10 +22,21 @@ import { traverseOperations } from "./traverseOperations.js"; })); console.log(inputs); - // TODO: This can also be run serverside - const results = await traverseOperations(workflow.operations, inputs); + const traverse = traverseOperations(workflow.operations, inputs, Functions); + + let pdfResults; + let iteration; + while (true) { + iteration = await traverse.next(); + if (iteration.done) { + pdfResults = iteration.value; + console.log(`data: processing done\n\n`); + break; + } + console.log(`data: ${iteration.value}\n\n`); + } - results.forEach(result => { + pdfResults.forEach(result => { download(result.buffer, result.fileName, "application/pdf"); }); }); diff --git a/public/traverseOperations.js b/public/traverseOperations.js index 47ed301e..c9a0c455 100644 --- a/public/traverseOperations.js +++ b/public/traverseOperations.js @@ -1,207 +1,133 @@ -import { extractPages } from "./functions/extractPages.js"; -import { impose } from "./functions/impose.js"; -import { mergePDFs } from "./functions/mergePDFs.js"; -import { rotatePages } from "./functions/rotatePDF.js"; -import { splitPDF } from "./functions/splitPDF.js"; import { organizeWaitOperations } from "./organizeWaitOperations.js"; -// TODO: Make this run with feedback like the server side func & The serverside func also got now functionality & fixes for split, so copy it and make it work with the new readsteam system on frontend -export async function traverseOperations(operations, input) { +export async function * traverseOperations(operations, input, Functions) { const waitOperations = organizeWaitOperations(operations); - const results = []; - await nextOperation(operations, input); + let results = []; + yield* nextOperation(operations, input); + console.log("Done2"); return results; - async function nextOperation(operations, input) { + async function * nextOperation(operations, input) { if(Array.isArray(operations) && operations.length == 0) { // isEmpty - console.log("operation done: " + input.fileName); - results.push(input); - return; + if(Array.isArray(input)) { + console.log("operation done: " + input[0].fileName + input.length > 1 ? "+" : ""); + results = results.concat(input); + return; + } + else { + console.log("operation done: " + input.fileName); + results.push(input); + return; + } } for (let i = 0; i < operations.length; i++) { - await computeOperation(operations[i], structuredClone(input)); // break references + yield* computeOperation(operations[i], structuredClone(input)); } } - async function computeOperation(operation, input) { + async function * computeOperation(operation, input) { + yield "Starting: " + operation.type; switch (operation.type) { - case "done": - console.log("Done operation will get called if all waits are done. Skipping for now.") + case "done": // Skip this, because it is a valid node. break; case "wait": const waitOperation = waitOperations[operation.values.id]; - waitOperation.input.push(input); + + if(Array.isArray(input)) { + waitOperation.input.concat(input); // TODO: May have unexpected concequences. Needs further testing! + } + else { + waitOperation.input.push(input); + } + waitOperation.waitCount--; if(waitOperation.waitCount == 0) { - await nextOperation(waitOperation.doneOperation.operations, waitOperation.input); - } - break; - case "removeObjects": - console.warn("RemoveObjects not implemented yet.") - - if(Array.isArray(input)) { - for (let i = 0; i < input.length; i++) { - // TODO: modfiy input - input[i].fileName += "_removedObjects"; - await nextOperation(operation.operations, input[i]); - } - } - else { - // TODO: modfiy input - input.fileName += "_removedObjects"; - await nextOperation(operation.operations, input); + yield* nextOperation(waitOperation.doneOperation.operations, waitOperation.input); } break; case "extract": - if(Array.isArray(input)) { - for (let i = 0; i < input.length; i++) { - input[i].fileName += "_extractedPages"; - input[i].buffer = await extractPages(input[i].buffer, operation.values["pagesToExtractArray"]); - await nextOperation(operation.operations, input[i]); - } - } - else { + yield* nToN(input, operation, async (input) => { input.fileName += "_extractedPages"; - input.buffer = await extractPages(input.buffer, operation.values["pagesToExtractArray"]); - await nextOperation(operation.operations, input); - } - break; - case "split": - // TODO: When a split goes into a wait function it might break the done condition, as it will count multiplpe times. - if(Array.isArray(input)) { - for (let i = 0; i < input.length; i++) { - const splits = await splitPDF(input[i].buffer, operation.values["pagesToSplitAfterArray"]); - - for (let j = 0; j < splits.length; j++) { - const split = {}; - split.originalFileName = input[i].originalFileName; - split.fileName = input[i].fileName + "_split"; - split.buffer = splits[j]; - await nextOperation(operation.operations, split); - } - } - } - else { - const splits = await splitPDF(input.buffer, operation.values["pagesToSplitAfterArray"]); - - for (let j = 0; j < splits.length; j++) { - const split = {}; - split.originalFileName = input.originalFileName; - split.fileName = input.fileName + "_split"; - split.buffer = splits[j]; - await nextOperation(operation.operations, split); - } - } - break; - case "fillField": - console.warn("FillField not implemented yet.") - - if(Array.isArray(input)) { - for (let i = 0; i < input.length; i++) { - // TODO: modfiy input - input[i].fileName += "_filledField"; - await nextOperation(operation.operations, input[i]); - } - } - else { - // TODO: modfiy input - input.fileName += "_filledField"; - await nextOperation(operation.operations, input); - } - break; - case "extractImages": - console.warn("ExtractImages not implemented yet.") - - if(Array.isArray(input)) { - for (let i = 0; i < input.length; i++) { - // TODO: modfiy input - input[i].fileName += "_extractedImages"; - await nextOperation(operation.operations, input[i]); - } - } - else { - // TODO: modfiy input - input.fileName += "_extractedImages"; - await nextOperation(operation.operations, input); - } - break; - case "merge": - if(Array.isArray(input) && input.length > 1) { - const inputs = input; - input = { - originalFileName: inputs.map(input => input.originalFileName).join("_and_"), - fileName: inputs.map(input => input.fileName).join("_and_") + "_merged", - buffer: await mergePDFs(inputs.map(input => input.buffer)) - } - } - else { - // Only one input, no need to merge - input.fileName += "_merged"; - } - await nextOperation(operation.operations, input); - break; - case "transform": { - console.warn("Transform not implemented yet.") - if(Array.isArray(input)) { - for (let i = 0; i < input.length; i++) { - // TODO: modfiy input - input[i].fileName += "_transformed"; - await nextOperation(operation.operations, input[i]); - } - } - else { - // TODO: modfiy input - input.fileName += "_transformed"; - await nextOperation(operation.operations, input); - } - break; - } - case "extract": - if(Array.isArray(input)) { - for (let i = 0; i < input.length; i++) { - input[i].fileName += "_extractedPages"; - input[i].buffer = await extractPages(input[i].buffer, operation.values["pagesToExtractArray"]); - await nextOperation(operation.operations, input[i]); - } - } - else { - input.fileName += "_extractedPages"; - input.buffer = await extractPages(input.buffer, operation.values["pagesToExtractArray"]); - await nextOperation(operation.operations, input); - } - break; - case "rotate": - if(Array.isArray(input)) { - for (let i = 0; i < input.length; i++) { - input[i].fileName += "_turned"; - input[i].buffer = await rotatePages(input[i].buffer, operation.values["rotation"]); - await nextOperation(operation.operations, input[i]); - } - } - else { - input.fileName += "_turned"; - input.buffer = await rotatePages(input.buffer, operation.values["rotation"]); - await nextOperation(operation.operations, input); - } + input.buffer = await Functions.extractPages(input.buffer, operation.values["pagesToExtractArray"]); + }); break; case "impose": - if(Array.isArray(input)) { - for (let i = 0; i < input.length; i++) { - input[i].fileName += "_imposed"; - input[i].buffer = await impose(input[i].buffer, operation.values["nup"], operation.values["format"]); - await nextOperation(operation.operations, input[i]); - } - } - else { + yield* nToN(input, operation, async (input) => { input.fileName += "_imposed"; - input.buffer = await impose(input.buffer, operation.values["nup"], operation.values["format"]); - await nextOperation(operation.operations, input); - } + input.buffer = await Functions.impose(input.buffer, operation.values["nup"], operation.values["format"]); + }); + break; + case "merge": + yield* nToOne(input, operation, async (inputs) => { + return { + originalFileName: inputs.map(input => input.originalFileName).join("_and_"), + fileName: inputs.map(input => input.fileName).join("_and_") + "_merged", + buffer: await Functions.mergePDFs(inputs.map(input => input.buffer)) + } + }); + break; + case "rotate": + yield* nToN(input, operation, async (input) => { + input.fileName += "_turned"; + input.buffer = await Functions.rotatePages(input.buffer, operation.values["rotation"]); + }); + break; + case "split": + // TODO: A split might break the done condition, it may count multiple times. Needs further testing! + yield* oneToN(input, operation, async (input) => { + const splitResult = await Functions.splitPDF(input.buffer, operation.values["pagesToSplitAfterArray"]); + + const splits = []; + for (let j = 0; j < splitResult.length; j++) { + splits.push({ + originalFileName: input.originalFileName, + fileName: input.fileName + "_split" + j, + buffer: splitResult[j] + }) + } + + input = splits; + }); break; default: - console.log("operation type unknown: ", operation.type); + throw new Error(`${operation.type} not implemented yet.`); break; } } + + async function * nToOne(inputs, operation, callback) { + if(!Array.isArray(inputs)) { + inputs = [inputs]; + } + + inputs = await callback(inputs); + yield* nextOperation(operation.operations, inputs); + } + + async function * oneToN(input, operation, callback) { + if(Array.isArray(input)) { + for (let i = 0; i < input.length; i++) { + await callback(input[i]); + } + yield* nextOperation(operation.operations, input); + } + else { + await callback(input); + yield* nextOperation(operation.operations, input); + } + } + + async function * nToN(input, operation, callback) { + if(Array.isArray(input)) { + for (let i = 0; i < input.length; i++) { + await callback(input[i]); + } + yield* nextOperation(operation.operations, input); + } + else { + await callback(input); + yield* nextOperation(operation.operations, input); + } + } } \ No newline at end of file diff --git a/public/wasm/pdfcpu-wrapper-browser.js b/public/wasm/pdfcpu-wrapper-browser.js index 44a94023..b0cbad7c 100644 --- a/public/wasm/pdfcpu-wrapper-browser.js +++ b/public/wasm/pdfcpu-wrapper-browser.js @@ -75,7 +75,7 @@ export async function impose(snapshot, nup, format) { export async function oneToOne(wasmArray, snapshot) { await loadFileAsync(Buffer.from(snapshot)); - console.error("Nuping File"); + console.log("Nuping File"); let exitcode = await runWasm(wasmArray); if (exitcode !== 0) { diff --git a/routes/api/workflow.js b/routes/api/workflow.js index a6a1465b..499484d4 100644 --- a/routes/api/workflow.js +++ b/routes/api/workflow.js @@ -3,7 +3,8 @@ import crypto from 'crypto'; import stream from "stream"; import Archiver from 'archiver'; -import { traverseOperations } from "../../traverseOperations.js"; +import * as Functions from "../../functions.js"; +import { traverseOperations } from "../../public/traverseOperations.js"; const activeWorkflows = {}; @@ -38,7 +39,7 @@ router.post("/:workflowUuid?", [ if(req.body.async === "false") { console.log("Don't do async"); - const traverse = traverseOperations(workflow.operations, inputs); + const traverse = traverseOperations(workflow.operations, inputs, Functions); let pdfResults; let iteration; diff --git a/traverseOperations.js b/traverseOperations.js deleted file mode 100644 index a7325846..00000000 --- a/traverseOperations.js +++ /dev/null @@ -1,134 +0,0 @@ -import * as Functions from "./functions/index.js"; -import { organizeWaitOperations } from "./public/organizeWaitOperations.js"; - -export async function * traverseOperations(operations, input) { - const waitOperations = organizeWaitOperations(operations); - let results = []; - yield* nextOperation(operations, input); - console.log("Done2"); - return results; - - async function * nextOperation(operations, input) { - if(Array.isArray(operations) && operations.length == 0) { // isEmpty - if(Array.isArray(input)) { - console.log("operation done: " + input[0].fileName + input.length > 1 ? "+" : ""); - results = results.concat(input); - return; - } - else { - console.log("operation done: " + input.fileName); - results.push(input); - return; - } - } - - for (let i = 0; i < operations.length; i++) { - yield* computeOperation(operations[i], structuredClone(input)); - } - } - - async function * computeOperation(operation, input) { - yield "Starting: " + operation.type; - switch (operation.type) { - case "done": // Skip this, because it is a valid node. - break; - case "wait": - const waitOperation = waitOperations[operation.values.id]; - - if(Array.isArray(input)) { - waitOperation.input.concat(input); // TODO: May have unexpected concequences. Needs further testing! - } - else { - waitOperation.input.push(input); - } - - waitOperation.waitCount--; - if(waitOperation.waitCount == 0) { - yield* nextOperation(waitOperation.doneOperation.operations, waitOperation.input); - } - break; - case "extract": - yield* nToN(input, operation, async (input) => { - input.fileName += "_extractedPages"; - input.buffer = await Functions.extractPages(input.buffer, operation.values["pagesToExtractArray"]); - }); - break; - case "impose": - yield* nToN(input, operation, async (input) => { - input.fileName += "_imposed"; - input.buffer = await Functions.impose(input.buffer, operation.values["nup"], operation.values["format"]); - }); - break; - case "merge": - yield* nToOne(input, operation, async (inputs) => { - return { - originalFileName: inputs.map(input => input.originalFileName).join("_and_"), - fileName: inputs.map(input => input.fileName).join("_and_") + "_merged", - buffer: await Functions.mergePDFs(inputs.map(input => input.buffer)) - } - }); - break; - case "rotate": - yield* nToN(input, operation, async (input) => { - input.fileName += "_turned"; - input.buffer = await Functions.rotatePages(input.buffer, operation.values["rotation"]); - }); - break; - case "split": - // TODO: A split might break the done condition, it may count multiple times. Needs further testing! - yield* oneToN(input, operation, async (input) => { - const splitResult = await Functions.splitPDF(input.buffer, operation.values["pagesToSplitAfterArray"]); - - const splits = []; - for (let j = 0; j < splitResult.length; j++) { - splits.push({ - originalFileName: input.originalFileName, - fileName: input.fileName + "_split" + j, - buffer: splitResult[j] - }) - } - - input = splits; - }); - break; - default: - throw new Error(`${operation.type} not implemented yet.`); - break; - } - } - - async function * nToOne(inputs, operation, callback) { - if(!Array.isArray(inputs)) { - inputs = [inputs]; - } - - inputs = await callback(inputs); - yield* nextOperation(operation.operations, inputs); - } - - async function * oneToN(input, operation, callback) { - if(Array.isArray(input)) { - for (let i = 0; i < input.length; i++) { - await callback(input[i]); - } - yield* nextOperation(operation.operations, input); - } - else { - await callback(input); - yield* nextOperation(operation.operations, input); - } - } - - async function * nToN(input, operation, callback) { - if(Array.isArray(input)) { - for (let i = 0; i < input.length; i++) { - await callback(input[i]); - } - yield* nextOperation(operation.operations, input); - } - else { - await callback(input); - yield* nextOperation(operation.operations, input); - } - } -} \ No newline at end of file