Add pipeline metrics, Fix pipeline Modal, zip fixed for null accepts

src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineController.java

@@ -5,6 +5,7 @@ import java.io.InputStream;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
 
@@ -26,8 +27,10 @@ import io.swagger.v3.oas.annotations.tags.Tag;
 import lombok.extern.slf4j.Slf4j;
 
 import stirling.software.SPDF.model.PipelineConfig;
+import stirling.software.SPDF.model.PipelineOperation;
 import stirling.software.SPDF.model.PipelineResult;
 import stirling.software.SPDF.model.api.HandleDataRequest;
+import stirling.software.SPDF.service.PostHogService;
 import stirling.software.SPDF.utils.WebResponseUtils;
 
 @RestController
@@ -40,9 +43,13 @@ public class PipelineController {
 
     private final ObjectMapper objectMapper;
 
-    public PipelineController(PipelineProcessor processor, ObjectMapper objectMapper) {
+    private final PostHogService postHogService;
+    
+    public PipelineController(PipelineProcessor processor, ObjectMapper objectMapper,
+            PostHogService postHogService) {
         this.processor = processor;
         this.objectMapper = objectMapper;
+        this.postHogService = postHogService;
     }
 
     @PostMapping("/handleData")
@@ -55,6 +62,18 @@ public class PipelineController {
         }
         PipelineConfig config = objectMapper.readValue(jsonString, PipelineConfig.class);
         log.info("Received POST request to /handleData with {} files", files.length);
+        
+        
+        List<String> operationNames = config.getOperations().stream()
+                .map(PipelineOperation::getOperation)
+                .collect(Collectors.toList());
+        
+        Map<String, Object> properties = new HashMap<>();
+        properties.put("operations", operationNames);
+        properties.put("fileCount", files.length);
+        
+        postHogService.captureEvent("pipeline_api_event", properties);
+        
         try {
             List<Resource> inputFiles = processor.generateInputFiles(files);
             if (inputFiles == null || inputFiles.size() == 0) {

src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineDirectoryProcessor.java

@@ -17,8 +17,11 @@ import java.time.LocalDate;
 import java.time.LocalTime;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import org.springframework.core.io.ByteArrayResource;
@@ -34,6 +37,7 @@ import stirling.software.SPDF.config.RuntimePathConfig;
 import stirling.software.SPDF.model.PipelineConfig;
 import stirling.software.SPDF.model.PipelineOperation;
 import stirling.software.SPDF.model.PipelineResult;
+import stirling.software.SPDF.service.PostHogService;
 import stirling.software.SPDF.utils.FileMonitor;
 
 @Service
@@ -41,15 +45,11 @@ import stirling.software.SPDF.utils.FileMonitor;
 public class PipelineDirectoryProcessor {
 
     private final ObjectMapper objectMapper;
-
     private final ApiDocService apiDocService;
-
     private final PipelineProcessor processor;
-
     private final FileMonitor fileMonitor;
-
+    private final PostHogService postHogService;
     private final String watchedFoldersDir;
-
     private final String finishedFoldersDir;
 
     public PipelineDirectoryProcessor(
@@ -57,13 +57,15 @@ public class PipelineDirectoryProcessor {
             ApiDocService apiDocService,
             PipelineProcessor processor,
             FileMonitor fileMonitor,
+            PostHogService postHogService,
             RuntimePathConfig runtimePathConfig) {
         this.objectMapper = objectMapper;
         this.apiDocService = apiDocService;
-        this.watchedFoldersDir = runtimePathConfig.getPipelineWatchedFoldersPath();
-        this.finishedFoldersDir = runtimePathConfig.getPipelineFinishedFoldersPath();
         this.processor = processor;
         this.fileMonitor = fileMonitor;
+        this.postHogService = postHogService;
+        this.watchedFoldersDir = runtimePathConfig.getPipelineWatchedFoldersPath();
+        this.finishedFoldersDir = runtimePathConfig.getPipelineFinishedFoldersPath();
     }
 
     @Scheduled(fixedRate = 60000)
@@ -152,6 +154,15 @@ public class PipelineDirectoryProcessor {
                 log.debug("No files detected for {} ", dir);
                 return;
             }
+            
+            List<String> operationNames = config.getOperations().stream()
+            	    .map(PipelineOperation::getOperation)
+            	    .toList(); 
+            Map<String, Object> properties = new HashMap<>();
+            properties.put("operations", operationNames);
+            properties.put("fileCount", files.length);
+            postHogService.captureEvent("pipeline_directory_event", properties);
+            
             List<File> filesToProcess = prepareFilesForProcessing(files, processingDir);
             runPipelineAgainstFiles(filesToProcess, config, dir, processingDir);
         }
@@ -252,9 +263,7 @@ public class PipelineDirectoryProcessor {
                         try {
                             Thread.sleep(retryDelayMs * (int) Math.pow(2, attempt - 1));
                         } catch (InterruptedException e1) {
-                            // TODO Auto-generated catch block
+                            log.error("prepareFilesForProcessing failure",e);                        }
-                            e1.printStackTrace();
-                        }
                     }
                 }
             }

src/main/resources/static/js/fileInput.js

@@ -134,7 +134,17 @@ function setupFileInput(chooser) {
       allFiles = Array.from(isDragAndDrop ? allFiles : [element.files[0]]);
     }
 
+	const originalText = inputContainer.querySelector('#fileInputText').innerHTML;
+
+	inputContainer.querySelector('#fileInputText').innerHTML = window.fileInput.loading;
+    
     async function checkZipFile() {
+      const hasZipFiles = allFiles.some(file => zipTypes.includes(file.type));
+      
+      // Only change to extractPDF message if we actually have zip files
+      if (hasZipFiles) {
+        inputContainer.querySelector('#fileInputText').innerHTML = window.fileInput.extractPDF;
+      }
 
       const promises = allFiles.map(async (file, index) => {
         try {
@@ -149,12 +159,9 @@ function setupFileInput(chooser) {
       });
 
       await Promise.all(promises);
-      
     }
-    const originalText = inputContainer.querySelector('#fileInputText').innerHTML;
-    const decryptFile = new DecryptFile();
     
-    inputContainer.querySelector('#fileInputText').innerHTML = window.fileInput.extractPDF;
+    const decryptFile = new DecryptFile();
 
     await checkZipFile();
 
@@ -217,26 +224,26 @@ function setupFileInput(chooser) {
       .then(function (zip) {
         var extractionPromises = [];
 
-        zip.forEach(function (relativePath, zipEntry) {
+       zip.forEach(function (relativePath, zipEntry) {
+		    const promise = zipEntry.async('blob').then(function (content) {
+		      // Assuming that folders have size zero
+		      if (content.size > 0) {
+		        const extension = zipEntry.name.split('.').pop().toLowerCase();
+		        const mimeType = mimeTypes[extension] || 'application/octet-stream';
 		
-          const promise = zipEntry.async('blob').then(function (content) {
+		        // Check if we're accepting ONLY ZIP files (in which case extract everything)
-            // Assuming that folders have size zero
+		        // or if the file type matches the accepted type
-            if (content.size > 0) {
+		        if (zipTypes.includes(acceptedFileType) || 
-              const extension = zipEntry.name.split('.').pop().toLowerCase();
+		            acceptedFileType === '*/*' || 
-              const mimeType = mimeTypes[extension];
+		            (mimeType && (mimeType.startsWith(acceptedFileType.split('/')[0]) || acceptedFileType === mimeType))) {
-  
+		          var file = new File([content], zipEntry.name, { type: mimeType });
-              // Check for file extension
+		          file.uniqueId = UUID.uuidv4();
-              if (mimeType && (mimeType.startsWith(acceptedFileType.split('/')[0]) || acceptedFileType === mimeType)) {
+		          allFiles.push(file);
-
+		        } else {
-                var file = new File([content], zipEntry.name, { type: mimeType });
+		          console.log(`File ${zipEntry.name} skipped. MIME type (${mimeType}) does not match accepted type (${acceptedFileType})`);
-                file.uniqueId = UUID.uuidv4();
+		        }
-                allFiles.push(file);
+		      }
-  
+		    });
-              } else {
-                console.log(`File ${zipEntry.name} skipped. MIME type (${mimeType}) does not match accepted type (${acceptedFileType})`);
-              }
-            }
-          });
   
           extractionPromises.push(promise);
         });

src/main/resources/templates/fragments/common.html

@@ -224,15 +224,20 @@
     window.fileInput = {
       dragAndDropPDF: '[[#{fileChooser.dragAndDropPDF}]]',
       dragAndDropImage: '[[#{fileChooser.dragAndDropImage}]]',
-      extractPDF: '[[#{fileChooser.extractPDF}]]'
+      extractPDF: '[[#{fileChooser.extractPDF}]]',
+      loading: '[[#{loading}]]'
     };</script>
   <div class="custom-file-chooser mb-3"
     th:attr="data-bs-unique-id=${name}, data-bs-element-id=${name+'-input'}, data-bs-element-container-id=${name+'-input-container'}, data-bs-files-selected=#{filesSelected}, data-bs-pdf-prompt=#{pdfPrompt}">
     <div class="mb-3 d-flex flex-row justify-content-center align-items-center flex-wrap input-container"
       th:name="${name}+'-input'" th:id="${name}+'-input-container'" th:data-text="#{fileChooser.hoveredDragAndDrop}">
       <label class="file-input-btn d-none">
-        <input type="file" class="form-control" th:name="${name}" th:id="${name}+'-input'" th:accept="${accept} + ',.zip'"
+       <input type="file" class="form-control" 
-          th:attr="multiple=${!disableMultipleFiles}" th:required="${notRequired} ? null : 'required'">
+		  th:name="${name}" 
+		  th:id="${name}+'-input'" 
+		  th:accept="${accept != null && accept != '*/*' ? accept + ',.zip' : (accept != null ? accept : '*/*')}"
+		  th:attr="multiple=${!disableMultipleFiles}" 
+		  th:required="${notRequired} ? null : 'required'">
         Browse
       </label>
       <div class="d-flex justify-content-start align-items-center" id="fileInputText">

src/main/resources/templates/pipeline.html

@@ -64,7 +64,7 @@
                   </div>
                   <div class="element-margin">
                     <div
-                      th:replace="~{fragments/common :: fileSelector(name='fileInput', multipleInputsForSingleRequest=true)}"
+                      th:replace="~{fragments/common :: fileSelector(name='fileInput', multipleInputsForSingleRequest=true, accept='*/*')}"
                     ></div>
                   </div>
                   <div class="element-margin text-start">
@@ -93,7 +93,7 @@
 
               <!-- The Modal -->
               <div class="modal" id="pipelineSettingsModal">
-                <div class="modal-dialog modal-lg">
+                <div class="modal-dialog modal-dialog-centered modal-lg">
                   <div class="modal-content dark-card">
                     <!-- Modal Header -->
                     <div class="modal-header">