diff --git a/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineController.java b/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineController.java index 291cd9b45..b1881d781 100644 --- a/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineController.java +++ b/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineController.java @@ -5,6 +5,7 @@ import java.io.InputStream; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -26,8 +27,10 @@ import io.swagger.v3.oas.annotations.tags.Tag; import lombok.extern.slf4j.Slf4j; import stirling.software.SPDF.model.PipelineConfig; +import stirling.software.SPDF.model.PipelineOperation; import stirling.software.SPDF.model.PipelineResult; import stirling.software.SPDF.model.api.HandleDataRequest; +import stirling.software.SPDF.service.PostHogService; import stirling.software.SPDF.utils.WebResponseUtils; @RestController @@ -40,9 +43,13 @@ public class PipelineController { private final ObjectMapper objectMapper; - public PipelineController(PipelineProcessor processor, ObjectMapper objectMapper) { + private final PostHogService postHogService; + + public PipelineController(PipelineProcessor processor, ObjectMapper objectMapper, + PostHogService postHogService) { this.processor = processor; this.objectMapper = objectMapper; + this.postHogService = postHogService; } @PostMapping("/handleData") @@ -55,6 +62,18 @@ public class PipelineController { } PipelineConfig config = objectMapper.readValue(jsonString, PipelineConfig.class); log.info("Received POST request to /handleData with {} files", files.length); + + + List operationNames = config.getOperations().stream() + .map(PipelineOperation::getOperation) + .collect(Collectors.toList()); + + Map properties = new HashMap<>(); + properties.put("operations", operationNames); + properties.put("fileCount", files.length); + + postHogService.captureEvent("pipeline_api_event", properties); + try { List inputFiles = processor.generateInputFiles(files); if (inputFiles == null || inputFiles.size() == 0) { diff --git a/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineDirectoryProcessor.java b/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineDirectoryProcessor.java index 192bed0e4..a83879e67 100644 --- a/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineDirectoryProcessor.java +++ b/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineDirectoryProcessor.java @@ -17,8 +17,11 @@ import java.time.LocalDate; import java.time.LocalTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; import java.util.stream.Stream; import org.springframework.core.io.ByteArrayResource; @@ -34,6 +37,7 @@ import stirling.software.SPDF.config.RuntimePathConfig; import stirling.software.SPDF.model.PipelineConfig; import stirling.software.SPDF.model.PipelineOperation; import stirling.software.SPDF.model.PipelineResult; +import stirling.software.SPDF.service.PostHogService; import stirling.software.SPDF.utils.FileMonitor; @Service @@ -41,15 +45,11 @@ import stirling.software.SPDF.utils.FileMonitor; public class PipelineDirectoryProcessor { private final ObjectMapper objectMapper; - private final ApiDocService apiDocService; - private final PipelineProcessor processor; - private final FileMonitor fileMonitor; - + private final PostHogService postHogService; private final String watchedFoldersDir; - private final String finishedFoldersDir; public PipelineDirectoryProcessor( @@ -57,13 +57,15 @@ public class PipelineDirectoryProcessor { ApiDocService apiDocService, PipelineProcessor processor, FileMonitor fileMonitor, + PostHogService postHogService, RuntimePathConfig runtimePathConfig) { this.objectMapper = objectMapper; this.apiDocService = apiDocService; - this.watchedFoldersDir = runtimePathConfig.getPipelineWatchedFoldersPath(); - this.finishedFoldersDir = runtimePathConfig.getPipelineFinishedFoldersPath(); this.processor = processor; this.fileMonitor = fileMonitor; + this.postHogService = postHogService; + this.watchedFoldersDir = runtimePathConfig.getPipelineWatchedFoldersPath(); + this.finishedFoldersDir = runtimePathConfig.getPipelineFinishedFoldersPath(); } @Scheduled(fixedRate = 60000) @@ -152,6 +154,15 @@ public class PipelineDirectoryProcessor { log.debug("No files detected for {} ", dir); return; } + + List operationNames = config.getOperations().stream() + .map(PipelineOperation::getOperation) + .toList(); + Map properties = new HashMap<>(); + properties.put("operations", operationNames); + properties.put("fileCount", files.length); + postHogService.captureEvent("pipeline_directory_event", properties); + List filesToProcess = prepareFilesForProcessing(files, processingDir); runPipelineAgainstFiles(filesToProcess, config, dir, processingDir); } @@ -252,9 +263,7 @@ public class PipelineDirectoryProcessor { try { Thread.sleep(retryDelayMs * (int) Math.pow(2, attempt - 1)); } catch (InterruptedException e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); - } + log.error("prepareFilesForProcessing failure",e); } } } } @@ -298,7 +307,7 @@ public class PipelineDirectoryProcessor { processor.generateInputFiles(filesToProcess.toArray(new File[0])); if (inputFiles == null || inputFiles.isEmpty()) { return; - } + } PipelineResult result = processor.runPipelineAgainstFiles(inputFiles, config); if (result.isHasErrors()) { @@ -395,4 +404,4 @@ public class PipelineDirectoryProcessor { } } } -} +} \ No newline at end of file diff --git a/src/main/resources/static/js/fileInput.js b/src/main/resources/static/js/fileInput.js index 91ad372ce..e2abb41c8 100644 --- a/src/main/resources/static/js/fileInput.js +++ b/src/main/resources/static/js/fileInput.js @@ -134,7 +134,17 @@ function setupFileInput(chooser) { allFiles = Array.from(isDragAndDrop ? allFiles : [element.files[0]]); } + const originalText = inputContainer.querySelector('#fileInputText').innerHTML; + + inputContainer.querySelector('#fileInputText').innerHTML = window.fileInput.loading; + async function checkZipFile() { + const hasZipFiles = allFiles.some(file => zipTypes.includes(file.type)); + + // Only change to extractPDF message if we actually have zip files + if (hasZipFiles) { + inputContainer.querySelector('#fileInputText').innerHTML = window.fileInput.extractPDF; + } const promises = allFiles.map(async (file, index) => { try { @@ -149,13 +159,10 @@ function setupFileInput(chooser) { }); await Promise.all(promises); - } - const originalText = inputContainer.querySelector('#fileInputText').innerHTML; + const decryptFile = new DecryptFile(); - inputContainer.querySelector('#fileInputText').innerHTML = window.fileInput.extractPDF; - await checkZipFile(); allFiles = await Promise.all( @@ -217,26 +224,26 @@ function setupFileInput(chooser) { .then(function (zip) { var extractionPromises = []; - zip.forEach(function (relativePath, zipEntry) { - - const promise = zipEntry.async('blob').then(function (content) { - // Assuming that folders have size zero - if (content.size > 0) { - const extension = zipEntry.name.split('.').pop().toLowerCase(); - const mimeType = mimeTypes[extension]; - - // Check for file extension - if (mimeType && (mimeType.startsWith(acceptedFileType.split('/')[0]) || acceptedFileType === mimeType)) { - - var file = new File([content], zipEntry.name, { type: mimeType }); - file.uniqueId = UUID.uuidv4(); - allFiles.push(file); - - } else { - console.log(`File ${zipEntry.name} skipped. MIME type (${mimeType}) does not match accepted type (${acceptedFileType})`); - } - } - }); + zip.forEach(function (relativePath, zipEntry) { + const promise = zipEntry.async('blob').then(function (content) { + // Assuming that folders have size zero + if (content.size > 0) { + const extension = zipEntry.name.split('.').pop().toLowerCase(); + const mimeType = mimeTypes[extension] || 'application/octet-stream'; + + // Check if we're accepting ONLY ZIP files (in which case extract everything) + // or if the file type matches the accepted type + if (zipTypes.includes(acceptedFileType) || + acceptedFileType === '*/*' || + (mimeType && (mimeType.startsWith(acceptedFileType.split('/')[0]) || acceptedFileType === mimeType))) { + var file = new File([content], zipEntry.name, { type: mimeType }); + file.uniqueId = UUID.uuidv4(); + allFiles.push(file); + } else { + console.log(`File ${zipEntry.name} skipped. MIME type (${mimeType}) does not match accepted type (${acceptedFileType})`); + } + } + }); extractionPromises.push(promise); }); diff --git a/src/main/resources/templates/fragments/common.html b/src/main/resources/templates/fragments/common.html index 1e801d06f..ef2d9047f 100644 --- a/src/main/resources/templates/fragments/common.html +++ b/src/main/resources/templates/fragments/common.html @@ -224,15 +224,20 @@ window.fileInput = { dragAndDropPDF: '[[#{fileChooser.dragAndDropPDF}]]', dragAndDropImage: '[[#{fileChooser.dragAndDropImage}]]', - extractPDF: '[[#{fileChooser.extractPDF}]]' + extractPDF: '[[#{fileChooser.extractPDF}]]', + loading: '[[#{loading}]]' };
diff --git a/src/main/resources/templates/pipeline.html b/src/main/resources/templates/pipeline.html index a81d4b91f..34abed18b 100644 --- a/src/main/resources/templates/pipeline.html +++ b/src/main/resources/templates/pipeline.html @@ -64,7 +64,7 @@
@@ -93,7 +93,7 @@