Cleaned the server-node dir

Split express into different ports for easier migration
Typed page ops that had warnings
This commit is contained in:
Felix Kaspar 2023-11-08 02:24:16 +01:00
parent 0f35c77074
commit 97e4eab7bb
32 changed files with 119 additions and 494 deletions

View File

@ -1,65 +1,13 @@
import { scaleContent } from "./functions/scaleContent.js";
import { scalePage, PageSize } from "./functions/scalePage.js";
import * as exampleWorkflows from "./exampleWorkflows.js";
import { traverseOperations } from "./traverseOperations.js";
import * as Functions from "./functions.js";
import express from 'express';
const app = express();
const PORT = 80;
(async () => {
const workflowField = document.getElementById("workflow");
// Server Frontend TODO: Make this typescript compatible
app.use(express.static('../client-vanilla/public'));
app.use(express.static('../shared-operations'));
const dropdown = document.getElementById("pdfOptions");
// Clear existing options (if any)
dropdown.innerHTML = '';
// Iterate over the keys of the object and create an option for each key
for (const key in exampleWorkflows) {
const option = document.createElement('option');
option.value = key;
option.text = key;
dropdown.appendChild(option);
}
const loadButton = document.getElementById("loadButton");
loadButton.addEventListener("click", (e) => {
workflowField.value = JSON.stringify(exampleWorkflows[dropdown.value], null, 2);
});
loadButton.click();
const pdfFileInput = document.getElementById('pdfFile');
const doneButton = document.getElementById("doneButton");
doneButton.addEventListener('click', async (e) => {
console.log("Starting...");
const files = Array.from(pdfFileInput.files);
const inputs = await Promise.all(files.map(async file => {
return {
originalFileName: file.name.replace(/\.[^/.]+$/, ""),
fileName: file.name.replace(/\.[^/.]+$/, ""),
buffer: new Uint8Array(await file.arrayBuffer())
}
}));
console.log(inputs);
const workflow = JSON.parse(workflowField.value);
console.log(workflow);
const traverse = traverseOperations(workflow.operations, inputs, Functions);
let pdfResults;
let iteration;
while (true) {
iteration = await traverse.next();
if (iteration.done) {
pdfResults = iteration.value;
console.log(`data: processing done\n\n`);
break;
}
console.log(`data: ${iteration.value}\n\n`);
}
// TODO: Zip if wanted
pdfResults.forEach(result => {
download(result.buffer, result.fileName, "application/pdf");
});
});
})();
// serve
app.listen(PORT, function (err) {
if (err) console.log(err);
console.log(`http://localhost:${PORT}`);
});

View File

@ -0,0 +1,65 @@
import { scaleContent } from "./functions/scaleContent.js";
import { scalePage, PageSize } from "./functions/scalePage.js";
import * as exampleWorkflows from "./exampleWorkflows.js";
import { traverseOperations } from "./traverseOperations.js";
import * as Functions from "./functions.js";
(async () => {
const workflowField = document.getElementById("workflow");
const dropdown = document.getElementById("pdfOptions");
// Clear existing options (if any)
dropdown.innerHTML = '';
// Iterate over the keys of the object and create an option for each key
for (const key in exampleWorkflows) {
const option = document.createElement('option');
option.value = key;
option.text = key;
dropdown.appendChild(option);
}
const loadButton = document.getElementById("loadButton");
loadButton.addEventListener("click", (e) => {
workflowField.value = JSON.stringify(exampleWorkflows[dropdown.value], null, 2);
});
loadButton.click();
const pdfFileInput = document.getElementById('pdfFile');
const doneButton = document.getElementById("doneButton");
doneButton.addEventListener('click', async (e) => {
console.log("Starting...");
const files = Array.from(pdfFileInput.files);
const inputs = await Promise.all(files.map(async file => {
return {
originalFileName: file.name.replace(/\.[^/.]+$/, ""),
fileName: file.name.replace(/\.[^/.]+$/, ""),
buffer: new Uint8Array(await file.arrayBuffer())
}
}));
console.log(inputs);
const workflow = JSON.parse(workflowField.value);
console.log(workflow);
const traverse = traverseOperations(workflow.operations, inputs, Functions);
let pdfResults;
let iteration;
while (true) {
iteration = await traverse.next();
if (iteration.done) {
pdfResults = iteration.value;
console.log(`data: processing done\n\n`);
break;
}
console.log(`data: ${iteration.value}\n\n`);
}
// TODO: Zip if wanted
pdfResults.forEach(result => {
download(result.buffer, result.fileName, "application/pdf");
});
});
})();

24
package-lock.json generated
View File

@ -6956,6 +6956,17 @@
"node": ">= 0.10.0"
}
},
"node_modules/express-fileupload": {
"version": "1.4.2",
"resolved": "https://registry.npmjs.org/express-fileupload/-/express-fileupload-1.4.2.tgz",
"integrity": "sha512-vk+9cK595jP03T+YgoYPAebynVCZuUBtW1JkyJnitQnWzlONHdxdAIm9yo99V4viTEftq7MUfzuqmWyqWGzMIg==",
"dependencies": {
"busboy": "^1.6.0"
},
"engines": {
"node": ">=12.0.0"
}
},
"node_modules/extend": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
@ -9020,6 +9031,11 @@
"verror": "1.10.0"
}
},
"node_modules/jsqr": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/jsqr/-/jsqr-1.4.0.tgz",
"integrity": "sha512-dxLob7q65Xg2DvstYkRpkYtmKm2sPJ9oFhrhmudT1dZvNFFTlroai3AWSpLey/w5vMcLBXRgOJsbXpdN9HzU/A=="
},
"node_modules/jsx-ast-utils": {
"version": "3.3.5",
"resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.3.5.tgz",
@ -10148,6 +10164,11 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/opencv-wasm": {
"version": "4.3.0-10",
"resolved": "https://registry.npmjs.org/opencv-wasm/-/opencv-wasm-4.3.0-10.tgz",
"integrity": "sha512-EWmWLUzp2suoc6N44Y4ouWT85QwvShx23Q430R+lp6NyS828bjQn6mCgA3NJ6Z/S59aaTeeu+RhqPQIJIYld1w=="
},
"node_modules/optionator": {
"version": "0.9.3",
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz",
@ -13647,7 +13668,10 @@
"@wasmer/wasmfs": "^0.12.0",
"archiver": "^6.0.1",
"express": "^4.18.2",
"express-fileupload": "^1.4.2",
"jsqr": "^1.4.0",
"multer": "^1.4.5-lts.1",
"opencv-wasm": "^4.3.0-10",
"pdf-lib": "^1.17.1"
}
},

View File

@ -1,4 +0,0 @@
.env
/testFiles/
/ignore/
*.code-workspace

View File

@ -1,142 +0,0 @@
# StirlingPDF rewrite
This is the development repository for the new StirlingPDF backend. With the power of JS, WASM & GO this will provide almost all functionality SPDF can do currently directly on the client. For automation purposes this will still provide an API to automate your workflows.
## Try the new API!
[![Run in Postman](https://run.pstmn.io/button.svg)](https://documenter.getpostman.com/view/30633786/2s9YRB1Wto)
## Understanding Workflows
Workflows define how to apply operations to a PDF, including their order and relations with eachother.
Workflows can be created via the web-ui and then exported or, if you want to brag a bit, you can create the JSON object yourself.
### Basics
To create your own, you have to understand a few key features first. You can also look at more examples our github repository.
```json
{
"outputOptions": {
"zip": false
},
"operations": [
{
"type": "extract",
"values": {
"pagesToExtractArray": [0, 2]
},
"operations": []
}
]
}
```
The workflow above will extract the first (p\[0\]) and third (p\[2\]) page of the document.
You can also nest workflows like this:
```json
{
"outputOptions": {
"zip": false
},
"operations": [
{
"type": "extract",
"values": {
"pagesToExtractArray": [0, 2]
},
"operations": [
{
"type": "impose",
"values": {
"nup": 2, // 2 pages of the input document will be put on one page of the output document.
"format": "A4L" // A4L -> The page size of the Ouput will be an A4 in Landscape. You can also use other paper formats and "P" for portrait output.
},
"operations": []
}
]
}
]
}
```
If you look at it closely, you will see that the extract operation has another nested operation of the type impose. This workflow will produce a PDF with the 1st and 2nd page of the input on one single page.
### Advanced
If that is not enought for you usecase, there is also the possibility to connect operations with eachother.
You can also do different operations to produce two different output PDFs from one input.
If you are interested in learning about this, take a look at the Example workflows provided in the repository, ask on the discord, or wait for me to finish this documentation.
## Features
### Rewrite Roadmap
* [x] Client side PDF-Manipulation
* [x] Workflows
* [ ] Feature equivalent with S-PDF v1
* [ ] Stateful UI
* [ ] Node based editing of Workflows
* [ ] Propper auth using passportjs
### Functions
Current functions of spdf and their progress in this repo.
| Status | Feature | Description |
| ------ | ---------------------- | ----------- |
| ✔️ | Merge | |
| ✔️ | Split | |
| ✔️ | Rotate | |
| ✔️ | Multi-Page-Layout | |
| ✔️ | Adjust page size/scale | |
| ✔️ | Organize | |
| ✔️ | Change Metadata | |
| ❌ | Add Watermark | |
| Status | Feature | Description |
| ------ | --------------------------- | ----------- |
| ❌ | Remove Pages | |
| ❌ | Remove Blank Pages | |
| ❌ | Detect/Split Scanned photos | |
| Status | Feature | Description |
| ------ | ------------ | ----------- |
| ❌ | Repair | |
| ❌ | Compress | |
| ❌ | Flatten | |
| ❌ | Compare/Diff | |
| Status | Feature | Description |
| ------ | --------------------- | ----------- |
| ❌ | Sign | |
| ❌ | Sign with Certificate | |
| ❌ | Add Password | |
| ❌ | Remove Password | |
| ❌ | Change Permissions | |
| Status | Feature | Description |
| ------ | -------------- | ----------- |
| ❌ | Image to PDF | |
| ❌ | Add image | |
| ❌ | Extract Images | |
| ❌ | PDF to Image | |
| ❌ | OCR | |
| Status | Feature | Description |
| ------ | ------------------- | ----------- |
| ❌ | Convert file to PDF | |
| ❌ | PDF to Text/RTF | |
| ❌ | PDF to HTML | |
| ❌ | PDF to XML | |
✔️: Done, 🚧: Started Developement, ❌: Planned Feature
## Contribute
For initial instructions look at [CONTRIBUTE.md](./CONTRIBUTE.md)

View File

@ -1,29 +1,11 @@
import operations from './routes/api/operations.js';
import express from 'express';
const app = express();
const PORT = 8080;
// Static Middleware
app.use(express.static('./public'));
app.get('/', function (req, res, next) { // TODO: Use EJS?
res.render('home.ejs');
});
app.use("/api/operations", operations);
//app.use("/api/workflow", workflow);
// server-node: backend api
import api from './server-node/routes/api/index.js';
import api from './routes/api/index.js';
app.use("/api/", api);
// client-vanilla: frontend
app.use(express.static('./client-vanilla'));
app.use(express.static('./shared-operations'));
// serve
app.listen(PORT, function (err) {
if (err) console.log(err);

View File

@ -12,7 +12,10 @@
"@wasmer/wasmfs": "^0.12.0",
"archiver": "^6.0.1",
"express": "^4.18.2",
"express-fileupload": "^1.4.2",
"jsqr": "^1.4.0",
"multer": "^1.4.5-lts.1",
"opencv-wasm": "^4.3.0-10",
"pdf-lib": "^1.17.1"
},
"type": "module"

View File

@ -1,165 +0,0 @@
// JSON Representation of this Node Tree:
// https://discord.com/channels/1068636748814483718/1099390571493195898/1118192754103693483
// https://cdn.discordapp.com/attachments/1099390571493195898/1118192753759764520/image.png?ex=6537dba7&is=652566a7&hm=dc46820ef7c34bc37424794966c5f66f93ba0e15a740742c364d47d31ea119a9&
export const discordWorkflow = {
outputOptions: {
zip: false
},
operations: [
{
type: "extract",
values: { "index": "1" },
operations: [
{
type: "removeObjects",
values: { "objectNames": "photo, josh" },
operations: [
{
type: "wait",
values: { "id": 1 }
}
]
}
]
},
{
type: "extract",
values: { "index": "2-5" },
operations: [
{
type: "fillField",
values: { "objectName": "name", "inputValue": "Josh" },
operations: [
{
type: "wait",
values: { "id": 1 }
}
]
}
]
},
{
type: "done", // This gets called when the other merge-ops with the same id finish.
values: { "id": 1 },
operations: [
{
type: "merge",
values: {},
operations: []
}
]
},
{
type: "extractImages",
values: {},
operations: []
},
{
type: "merge",
values: {},
operations: [
{
type: "transform",
values: { "scale": "2x", "rotation": "90deg" },
operations: []
}
]
}
]
}
// This will merge all input files into one giant document
export const mergeOnly = {
outputOptions: {
zip: false
},
operations: [
{
type: "merge",
values: {},
operations: []
}
]
}
// Extract Pages and store them in a new document
export const extractOnly = {
outputOptions: {
zip: false
},
operations: [
{
type: "extract",
values: { "pagesToExtractArray": [0, 2] },
operations: []
}
]
}
// Split a document up into multiple documents
export const splitOnly = {
outputOptions: {
zip: false
},
operations: [
{
type: "split",
values: { "pagesToSplitAfterArray": [2, 10] },
operations: []
}
]
}
export const rotateOnly = {
outputOptions: {
zip: false
},
operations: [
{
type: "rotate",
values: { "rotation": -90 },
operations: []
}
]
}
export const imposeOnly = {
outputOptions: {
zip: false
},
operations: [
{
type: "impose",
values: { "nup": 2, "format": "A4L" },
operations: []
}
]
}
export const removeBlankPagesOnly = {
outputOptions: {
zip: false
},
operations: [
{
type: "removeBlankPages",
values: { "whiteThreashold": 10 },
operations: []
}
]
}
export const splitOnQR = {
outputOptions: {
zip: false
},
operations: [
{
type: "splitOn",
values: {
type: "QR_CODE"
},
operations: []
}
]
}

View File

@ -1,17 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Document</title>
</head>
<body>
<h1>Rotate</h1>
<form action="/rotate-pdf" method="post" enctype="multipart/form-data">
<input type="file" name="pdfFile" accept=".pdf" multiple>
<br>
<button id="doneButton">Done</button>
</form>
</body>
</html>

View File

@ -1,17 +0,0 @@
import { rotatePages } from '../../src/pdf-operations.js';
import { respondWithBinaryPdf } from '../../src/utils/endpoint-utils.js';
import express from 'express';
const router = express.Router();
import multer from 'multer'
const upload = multer();
router.post('/rotate-pdf', upload.single("pdfFile"), async function(req, res, next) {
console.debug("rotating pdf:", req.file)
const rotated = await rotatePages(req.file.buffer, 90)
const newFilename = req.file.originalname.replace(/(\.[^.]+)$/, '_rotated$1'); // add '_rotated' just before the file extension
respondWithBinaryPdf(res, rotated, newFilename);
});
export default router;

View File

@ -6,7 +6,7 @@ import multer from 'multer'
const upload = multer();
import * as Functions from "../../functions.js";
import { traverseOperations } from "../../../shared-operations/traverseOperations.js";
import { traverseOperations } from "../../../shared-operations/workflow/traverseOperations.js";
const activeWorkflows = {};

View File

@ -1,41 +0,0 @@
import { editMetadata as dependantEditMetadata } from '@stirling-pdf/shared-operations/functions/editMetadata.js';
import { extractPages as dependantExtractPages } from '@stirling-pdf/shared-operations/functions/extractPages.js';
import { mergePDFs as dependantMergePDFs } from '@stirling-pdf/shared-operations/functions/mergePDFs.js';
import { organizePages as dependantOrganizePages } from '@stirling-pdf/shared-operations/functions/organizePages.js';
import { rotatePages as dependantRotatePages } from '@stirling-pdf/shared-operations/functions/rotatePages.js';
import { scaleContent as dependantScaleContent} from '@stirling-pdf/shared-operations/functions/scaleContent.js';
import { scalePage as dependantScalePage } from '@stirling-pdf/shared-operations/functions/scalePage.js';
import { splitPDF as dependantSplitPDF } from '@stirling-pdf/shared-operations/functions/splitPDF.js';
export async function editMetadata(snapshot, metadata) {
return dependantEditMetadata(snapshot, metadata);
}
export async function extractPages(snapshot, pagesToExtractArray) {
return dependantExtractPages(snapshot, pagesToExtractArray);
}
export async function mergePDFs(snapshots) {
return dependantMergePDFs(snapshots);
}
export async function organizePages(snapshot, operation, customOrderString) {
return dependantOrganizePages(snapshot, operation, customOrderString);
}
export async function rotatePages(snapshot, rotation) {
return dependantRotatePages(snapshot, rotation);
}
export async function scaleContent(snapshot, scaleFactor) {
return dependantScaleContent(snapshot, scaleFactor);
}
export async function scalePage(snapshot, pageSize) {
return dependantScalePage(snapshot, pageSize);
}
export async function splitPDF(snapshot, splitAfterPageArray) {
return dependantSplitPDF(snapshot, splitAfterPageArray);
}

View File

@ -1,9 +0,0 @@
export function respondWithBinaryPdf(res, buffer, filename) {
res.writeHead(200, {
'Content-Type': "application/pdf",
'Content-disposition': 'attachment;filename=' + filename,
'Content-Length': buffer.length
});
res.end(Buffer.from(buffer, 'binary'))
}

View File

@ -1,6 +1,6 @@
import { PDFDocument } from 'pdf-lib';
import { createSubDocument } from './createSubDocument';
import { createSubDocument } from './common/createSubDocument';
export async function extractPages(snapshot: string | Uint8Array | ArrayBuffer, pagesToExtractArray: number[]): Promise<Uint8Array>{
const pdfDoc = await PDFDocument.load(snapshot)

View File

@ -1,5 +1,5 @@
import { PDFDocument } from 'pdf-lib';
import { PDFDocument, PDFPage } from 'pdf-lib';
export async function organizePages(
snapshot: string | Uint8Array | ArrayBuffer,
@ -29,14 +29,14 @@ export async function organizePages(
customOrderedPages.forEach((page) => subDocument.addPage(page));
break;
case "REVERSE_ORDER":
const reversedPages = [];
const reversedPages: PDFPage[] = [];
for (let i = pageCount - 1; i >= 0; i--) {
reversedPages.push(copiedPages[i]);
}
reversedPages.forEach((page) => subDocument.addPage(page));
break;
case 'DUPLEX_SORT': //TODO: Needs to be checked by someone who knows more about duplex printing.
const duplexPages = [];
const duplexPages: PDFPage[] = [];
const half = (pageCount + 1) / 2
for (let i = 1; i <= half; i++) {
duplexPages.push(copiedPages[i - 1]);
@ -47,7 +47,7 @@ export async function organizePages(
duplexPages.forEach((page) => subDocument.addPage(page));
break;
case 'BOOKLET_SORT':
const bookletPages = [];
const bookletPages: PDFPage[] = [];
for (let i = 0; i < pageCount / 2; i++) {
bookletPages.push(copiedPages[i]);
bookletPages.push(copiedPages[pageCount - i - 1]);
@ -55,8 +55,8 @@ export async function organizePages(
bookletPages.forEach((page) => subDocument.addPage(page));
break;
case 'ODD_EVEN_SPLIT':
const oddPages = [];
const evenPages = [];
const oddPages: PDFPage[] = [];
const evenPages: PDFPage[] = [];
for (let i = 0; i < pageCount; i++) {
if (i % 2 === 0) {
evenPages.push(copiedPages[i]);

View File

@ -1,5 +1,5 @@
import { PDFDocument } from 'pdf-lib';
import { detectEmptyPages } from "./detectEmptyPages.js";
import { detectEmptyPages } from "./common/detectEmptyPages.js";
export async function removeBlankPages(snapshot, whiteThreashold) {

View File

@ -2,9 +2,9 @@
import { PDFDocument } from 'pdf-lib';
import PDFJS from 'pdfjs-dist';
import { detectEmptyPages } from "./detectEmptyPages.js";
import { getImagesOnPage } from "./getImagesOnPage.js";
import { createSubDocument } from "./createSubDocument.js";
import { detectEmptyPages } from "./common/detectEmptyPages.js";
import { getImagesOnPage } from "./common/getImagesOnPage.js";
import { createSubDocument } from "./common/createSubDocument.js";
import { TypedArray, DocumentInitParameters } from 'pdfjs-dist/types/src/display/api.js';
export async function splitOn(

View File

@ -1,7 +1,7 @@
import { PDFDocument } from 'pdf-lib';
import { createSubDocument } from "./createSubDocument.js";
import { createSubDocument } from "./common/createSubDocument.js";
export async function splitPDF(snapshot: string | Uint8Array | ArrayBuffer, splitAfterPageArray: number[]): Promise<Uint8Array[]> {
@ -9,9 +9,9 @@ export async function splitPDF(snapshot: string | Uint8Array | ArrayBuffer, spli
const numberOfPages = pdfDoc.getPages().length;
let pagesArray = [];
let pagesArray: number[] = [];
let splitAfter = splitAfterPageArray.shift();
const subDocuments = [];
const subDocuments: Uint8Array[] = [];
for (let i = 0; i < numberOfPages; i++) {
if(splitAfter && i > splitAfter && pagesArray.length > 0) {

View File

@ -1,2 +0,0 @@
export {};