From dfd567b803641d60be14c58dae67802e726ece1c Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com.> Date: Tue, 25 Mar 2025 17:36:01 +0000 Subject: [PATCH] Fix PATH order for python3 calls, improve merge memory --- Dockerfile | 2 +- Dockerfile.fat | 2 +- .../SPDF/EE/KeygenLicenseVerifier.java | 11 +- .../SPDF/config/EndpointConfiguration.java | 6 +- .../SPDF/controller/api/MergeController.java | 32 +++-- .../api/converters/ConvertWebsiteToPDF.java | 10 +- .../controller/api/misc/OCRController.java | 2 +- .../controller/web/MetricsController.java | 130 ++++++++++++++---- .../service/CustomPDFDocumentFactory.java | 2 +- src/main/resources/settings.yml.template | 4 - .../converters/ConvertWebsiteToPdfTest.java | 8 +- 11 files changed, 142 insertions(+), 67 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1b294ea6c..3d8780bc4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,7 +34,7 @@ ENV DOCKER_ENABLE_SECURITY=false \ PYTHONPATH=/usr/lib/libreoffice/program:/opt/venv/lib/python3.12/site-packages \ UNO_PATH=/usr/lib/libreoffice/program \ URE_BOOTSTRAP=file:///usr/lib/libreoffice/program/fundamentalrc \ - PATH=/opt/venv/bin:$PATH + PATH=$PATH:/opt/venv/bin # JDK for app diff --git a/Dockerfile.fat b/Dockerfile.fat index d16661e79..738ab87d8 100644 --- a/Dockerfile.fat +++ b/Dockerfile.fat @@ -43,7 +43,7 @@ ENV DOCKER_ENABLE_SECURITY=false \ PYTHONPATH=/usr/lib/libreoffice/program:/opt/venv/lib/python3.12/site-packages \ UNO_PATH=/usr/lib/libreoffice/program \ URE_BOOTSTRAP=file:///usr/lib/libreoffice/program/fundamentalrc \ - PATH=/opt/venv/bin:$PATH + PATH=$PATH:/opt/venv/bin # JDK for app diff --git a/src/main/java/stirling/software/SPDF/EE/KeygenLicenseVerifier.java b/src/main/java/stirling/software/SPDF/EE/KeygenLicenseVerifier.java index b2b7c018e..28e0c7e7e 100644 --- a/src/main/java/stirling/software/SPDF/EE/KeygenLicenseVerifier.java +++ b/src/main/java/stirling/software/SPDF/EE/KeygenLicenseVerifier.java @@ -28,13 +28,13 @@ public class KeygenLicenseVerifier { // License verification configuration private static final String ACCOUNT_ID = "e5430f69-e834-4ae4-befd-b602aae5f372"; private static final String BASE_URL = "https://api.keygen.sh/v1/accounts"; - + private static final String PUBLIC_KEY = "9fbc0d78593dcfcf03c945146edd60083bf5fae77dbc08aaa3935f03ce94a58d"; - + private static final String CERT_PREFIX = "-----BEGIN LICENSE FILE-----"; private static final String CERT_SUFFIX = "-----END LICENSE FILE-----"; - + private static final String JWT_PREFIX = "key/"; private static final ObjectMapper objectMapper = new ObjectMapper(); @@ -77,8 +77,7 @@ public class KeygenLicenseVerifier { encodedPayload = encodedPayload.replace(CERT_SUFFIX, ""); // Remove all newlines encodedPayload = encodedPayload.replaceAll("\\r?\\n", ""); - - + byte[] payloadBytes = Base64.getDecoder().decode(encodedPayload); String payload = new String(payloadBytes); @@ -403,7 +402,7 @@ public class KeygenLicenseVerifier { return false; } } - + private boolean verifyStandardLicense(String licenseKey) { try { log.info("Checking standard license key"); diff --git a/src/main/java/stirling/software/SPDF/config/EndpointConfiguration.java b/src/main/java/stirling/software/SPDF/config/EndpointConfiguration.java index fd2aed5f7..9c2d26548 100644 --- a/src/main/java/stirling/software/SPDF/config/EndpointConfiguration.java +++ b/src/main/java/stirling/software/SPDF/config/EndpointConfiguration.java @@ -289,9 +289,9 @@ public class EndpointConfiguration { if (!runningEE) { disableGroup("enterprise"); } - - if(!applicationProperties.getSystem().getEnableUrlToPDF()) { - disableEndpoint("url-to-pdf"); + + if (!applicationProperties.getSystem().getEnableUrlToPDF()) { + disableEndpoint("url-to-pdf"); } } diff --git a/src/main/java/stirling/software/SPDF/controller/api/MergeController.java b/src/main/java/stirling/software/SPDF/controller/api/MergeController.java index c95aff8fe..bd2731b7e 100644 --- a/src/main/java/stirling/software/SPDF/controller/api/MergeController.java +++ b/src/main/java/stirling/software/SPDF/controller/api/MergeController.java @@ -125,8 +125,7 @@ public class MergeController { public ResponseEntity mergePdfs(@ModelAttribute MergePdfsRequest form) throws IOException { List filesToDelete = new ArrayList<>(); // List of temporary files to delete - ByteArrayOutputStream docOutputstream = - new ByteArrayOutputStream(); // Stream for the merged document + File mergedTempFile = null; PDDocument mergedDocument = null; boolean removeCertSign = form.isRemoveCertSign(); @@ -139,21 +138,24 @@ public class MergeController { form.getSortType())); // Sort files based on the given sort type PDFMergerUtility mergerUtility = new PDFMergerUtility(); + long totalSize = 0; for (MultipartFile multipartFile : files) { + totalSize += multipartFile.getSize(); File tempFile = GeneralUtils.convertMultipartFileToFile( multipartFile); // Convert MultipartFile to File filesToDelete.add(tempFile); // Add temp file to the list for later deletion mergerUtility.addSource(tempFile); // Add source file to the merger utility } - mergerUtility.setDestinationStream( - docOutputstream); // Set the output stream for the merged document - mergerUtility.mergeDocuments(null); // Merge the documents - - byte[] mergedPdfBytes = docOutputstream.toByteArray(); // Get merged document bytes + + mergedTempFile = File.createTempFile("merged-", ".pdf"); + mergerUtility.setDestinationFileName(mergedTempFile.getAbsolutePath()); + + mergerUtility.mergeDocuments( + pdfDocumentFactory.getStreamCacheFunction(totalSize)); // Merge the documents // Load the merged PDF document - mergedDocument = pdfDocumentFactory.load(mergedPdfBytes); + mergedDocument = pdfDocumentFactory.load(mergedTempFile); // Remove signatures if removeCertSign is true if (removeCertSign) { @@ -180,21 +182,23 @@ public class MergeController { String mergedFileName = files[0].getOriginalFilename().replaceFirst("[.][^.]+$", "") + "_merged_unsigned.pdf"; - return WebResponseUtils.bytesToWebResponse( - baos.toByteArray(), mergedFileName); // Return the modified PDF + return WebResponseUtils.boasToWebResponse( + baos, mergedFileName); // Return the modified PDF } catch (Exception ex) { log.error("Error in merge pdf process", ex); throw ex; } finally { + if (mergedDocument != null) { + mergedDocument.close(); // Close the merged document + } for (File file : filesToDelete) { if (file != null) { Files.deleteIfExists(file.toPath()); // Delete temporary files } - } - docOutputstream.close(); - if (mergedDocument != null) { - mergedDocument.close(); // Close the merged document + } + if (mergedTempFile != null) { + Files.deleteIfExists(mergedTempFile.toPath()); } } } diff --git a/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertWebsiteToPDF.java b/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertWebsiteToPDF.java index b7f737c07..941d55795 100644 --- a/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertWebsiteToPDF.java +++ b/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertWebsiteToPDF.java @@ -37,9 +37,12 @@ public class ConvertWebsiteToPDF { private final CustomPDFDocumentFactory pdfDocumentFactory; private final RuntimePathConfig runtimePathConfig; private final ApplicationProperties applicationProperties; + @Autowired public ConvertWebsiteToPDF( - CustomPDFDocumentFactory pdfDocumentFactory, RuntimePathConfig runtimePathConfig, ApplicationProperties applicationProperties) { + CustomPDFDocumentFactory pdfDocumentFactory, + RuntimePathConfig runtimePathConfig, + ApplicationProperties applicationProperties) { this.pdfDocumentFactory = pdfDocumentFactory; this.runtimePathConfig = runtimePathConfig; this.applicationProperties = applicationProperties; @@ -55,9 +58,8 @@ public class ConvertWebsiteToPDF { throws IOException, InterruptedException { String URL = request.getUrlInput(); - - if(!applicationProperties.getSystem().getEnableUrlToPDF()) { - throw new IllegalArgumentException("This endpoint has been disabled by the admin."); + if (!applicationProperties.getSystem().getEnableUrlToPDF()) { + throw new IllegalArgumentException("This endpoint has been disabled by the admin."); } // Validate the URL format if (!URL.matches("^https?://.*") || !GeneralUtils.isValidURL(URL)) { diff --git a/src/main/java/stirling/software/SPDF/controller/api/misc/OCRController.java b/src/main/java/stirling/software/SPDF/controller/api/misc/OCRController.java index 059e1f011..47d6d5b35 100644 --- a/src/main/java/stirling/software/SPDF/controller/api/misc/OCRController.java +++ b/src/main/java/stirling/software/SPDF/controller/api/misc/OCRController.java @@ -215,4 +215,4 @@ public class OCRController { log.error("Error walking directory {}: {}", directory, e.getMessage()); } } -} \ No newline at end of file +} diff --git a/src/main/java/stirling/software/SPDF/controller/web/MetricsController.java b/src/main/java/stirling/software/SPDF/controller/web/MetricsController.java index f9540b95d..aae09341e 100644 --- a/src/main/java/stirling/software/SPDF/controller/web/MetricsController.java +++ b/src/main/java/stirling/software/SPDF/controller/web/MetricsController.java @@ -22,6 +22,7 @@ import jakarta.annotation.PostConstruct; import lombok.extern.slf4j.Slf4j; +import stirling.software.SPDF.config.EndpointInspector; import stirling.software.SPDF.config.StartupApplicationListener; import stirling.software.SPDF.model.ApplicationProperties; @@ -32,15 +33,17 @@ import stirling.software.SPDF.model.ApplicationProperties; public class MetricsController { private final ApplicationProperties applicationProperties; - private final MeterRegistry meterRegistry; - + private final EndpointInspector endpointInspector; private boolean metricsEnabled; public MetricsController( - ApplicationProperties applicationProperties, MeterRegistry meterRegistry) { + ApplicationProperties applicationProperties, + MeterRegistry meterRegistry, + EndpointInspector endpointInspector) { this.applicationProperties = applicationProperties; this.meterRegistry = meterRegistry; + this.endpointInspector = endpointInspector; } @PostConstruct @@ -208,16 +211,40 @@ public class MetricsController { } private double getRequestCount(String method, Optional endpoint) { - double count = - meterRegistry.find("http.requests").tag("method", method).counters().stream() - .filter( - counter -> - !endpoint.isPresent() - || endpoint.get() - .equals(counter.getId().getTag("uri"))) - .mapToDouble(Counter::count) - .sum(); - return count; + return meterRegistry.find("http.requests").tag("method", method).counters().stream() + .filter( + counter -> { + String uri = counter.getId().getTag("uri"); + + // Apply filtering logic - Skip if uri is null + if (uri == null) { + return false; + } + + // For POST requests, only include if they start with /api/v1 + if ("POST".equals(method) && !uri.contains("api/v1")) { + return false; + } + + if (uri.contains(".txt")) { + return false; + } + + // For GET requests, validate if we have a list of valid endpoints + final boolean validateGetEndpoints = + endpointInspector.getValidGetEndpoints().size() != 0; + if ("GET".equals(method) + && validateGetEndpoints + && !endpointInspector.isValidGetEndpoint(uri)) { + log.debug("Skipping invalid GET endpoint: {}", uri); + return false; + } + + // Filter for specific endpoint if provided + return !endpoint.isPresent() || endpoint.get().equals(uri); + }) + .mapToDouble(Counter::count) + .sum(); } private List getEndpointCounts(String method) { @@ -229,23 +256,72 @@ public class MetricsController { .forEach( counter -> { String uri = counter.getId().getTag("uri"); + + // Skip if uri is null + if (uri == null) { + return; + } + + // For POST requests, only include if they start with /api/v1 + if ("POST".equals(method) && !uri.contains("api/v1")) { + return; + } + + if (uri.contains(".txt")) { + return; + } + + // For GET requests, validate if we have a list of valid endpoints + final boolean validateGetEndpoints = + endpointInspector.getValidGetEndpoints().size() != 0; + if ("GET".equals(method) + && validateGetEndpoints + && !endpointInspector.isValidGetEndpoint(uri)) { + log.debug("Skipping invalid GET endpoint: {}", uri); + return; + } + counts.merge(uri, counter.count(), Double::sum); }); - List result = - counts.entrySet().stream() - .map(entry -> new EndpointCount(entry.getKey(), entry.getValue())) - .sorted(Comparator.comparing(EndpointCount::getCount).reversed()) - .collect(Collectors.toList()); - return result; + + return counts.entrySet().stream() + .map(entry -> new EndpointCount(entry.getKey(), entry.getValue())) + .sorted(Comparator.comparing(EndpointCount::getCount).reversed()) + .collect(Collectors.toList()); } private double getUniqueUserCount(String method, Optional endpoint) { Set uniqueUsers = new HashSet<>(); meterRegistry.find("http.requests").tag("method", method).counters().stream() .filter( - counter -> - !endpoint.isPresent() - || endpoint.get().equals(counter.getId().getTag("uri"))) + counter -> { + String uri = counter.getId().getTag("uri"); + + // Skip if uri is null + if (uri == null) { + return false; + } + + // For POST requests, only include if they start with /api/v1 + if ("POST".equals(method) && !uri.contains("api/v1")) { + return false; + } + + if (uri.contains(".txt")) { + return false; + } + + // For GET requests, validate if we have a list of valid endpoints + final boolean validateGetEndpoints = + endpointInspector.getValidGetEndpoints().size() != 0; + if ("GET".equals(method) + && validateGetEndpoints + && !endpointInspector.isValidGetEndpoint(uri)) { + log.debug("Skipping invalid GET endpoint: {}", uri); + return false; + } + return !endpoint.isPresent() || endpoint.get().equals(uri); + }) .forEach( counter -> { String session = counter.getId().getTag("session"); @@ -270,12 +346,10 @@ public class MetricsController { uniqueUsers.computeIfAbsent(uri, k -> new HashSet<>()).add(session); } }); - List result = - uniqueUsers.entrySet().stream() - .map(entry -> new EndpointCount(entry.getKey(), entry.getValue().size())) - .sorted(Comparator.comparing(EndpointCount::getCount).reversed()) - .collect(Collectors.toList()); - return result; + return uniqueUsers.entrySet().stream() + .map(entry -> new EndpointCount(entry.getKey(), entry.getValue().size())) + .sorted(Comparator.comparing(EndpointCount::getCount).reversed()) + .collect(Collectors.toList()); } @GetMapping("/uptime") diff --git a/src/main/java/stirling/software/SPDF/service/CustomPDFDocumentFactory.java b/src/main/java/stirling/software/SPDF/service/CustomPDFDocumentFactory.java index 92055a76c..b9ccd7b53 100644 --- a/src/main/java/stirling/software/SPDF/service/CustomPDFDocumentFactory.java +++ b/src/main/java/stirling/software/SPDF/service/CustomPDFDocumentFactory.java @@ -139,7 +139,7 @@ public class CustomPDFDocumentFactory { * Determine the appropriate caching strategy based on file size and available memory. This * common method is used by both password and non-password loading paths. */ - private StreamCacheCreateFunction getStreamCacheFunction(long contentSize) { + public StreamCacheCreateFunction getStreamCacheFunction(long contentSize) { long maxMemory = Runtime.getRuntime().maxMemory(); long freeMemory = Runtime.getRuntime().freeMemory(); long totalMemory = Runtime.getRuntime().totalMemory(); diff --git a/src/main/resources/settings.yml.template b/src/main/resources/settings.yml.template index b53cb30a8..b7306861c 100644 --- a/src/main/resources/settings.yml.template +++ b/src/main/resources/settings.yml.template @@ -72,10 +72,6 @@ premium: author: username creator: Stirling-PDF producer: Stirling-PDF - enterpriseFeatures: - persistentMetrics: - enabled: false - retentionDays: 90 legal: termsAndConditions: https://www.stirlingpdf.com/terms-and-conditions # URL to the terms and conditions of your application (e.g. https://example.com/terms). Empty string to disable or filename to load from local file in static folder diff --git a/src/test/java/stirling/software/SPDF/controller/api/converters/ConvertWebsiteToPdfTest.java b/src/test/java/stirling/software/SPDF/controller/api/converters/ConvertWebsiteToPdfTest.java index 8dbeb2515..a67e84f60 100644 --- a/src/test/java/stirling/software/SPDF/controller/api/converters/ConvertWebsiteToPdfTest.java +++ b/src/test/java/stirling/software/SPDF/controller/api/converters/ConvertWebsiteToPdfTest.java @@ -2,7 +2,6 @@ package stirling.software.SPDF.controller.api.converters; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.when; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -21,7 +20,7 @@ public class ConvertWebsiteToPdfTest { @Mock private RuntimePathConfig runtimePathConfig; private ApplicationProperties applicationProperties; - + private ConvertWebsiteToPDF convertWebsiteToPDF; @BeforeEach @@ -29,13 +28,14 @@ public class ConvertWebsiteToPdfTest { MockitoAnnotations.openMocks(this); applicationProperties = new ApplicationProperties(); applicationProperties.getSystem().setEnableUrlToPDF(true); - convertWebsiteToPDF = new ConvertWebsiteToPDF(mockPdfDocumentFactory, runtimePathConfig, applicationProperties); + convertWebsiteToPDF = + new ConvertWebsiteToPDF( + mockPdfDocumentFactory, runtimePathConfig, applicationProperties); } @Test public void test_exemption_is_thrown_when_invalid_url_format_provided() { - String invalid_format_Url = "invalid-url"; UrlToPdfRequest request = new UrlToPdfRequest();