This commit is contained in:
Anthony Stirling 2025-06-24 00:06:35 +01:00
parent 3e5dc34bbc
commit 09a1f9b1a1
13 changed files with 530 additions and 266 deletions

View File

@ -2,7 +2,10 @@
"permissions": { "permissions": {
"allow": [ "allow": [
"Bash(chmod:*)", "Bash(chmod:*)",
"Bash(mkdir:*)" "Bash(mkdir:*)",
"Bash(./gradlew:*)",
"Bash(grep:*)",
"Bash(cat:*)"
], ],
"deny": [] "deny": []
} }

View File

@ -8,22 +8,22 @@ import org.springframework.web.bind.annotation.RequestMethod;
/** /**
* Shortcut for a POST endpoint that is executed through the Stirling "autojob" framework. * Shortcut for a POST endpoint that is executed through the Stirling "autojob" framework.
* <p>
* Behaviour notes:
* <ul>
* <li>The endpoint is registered with {@code POST} and, by default, consumes
* {@code multipart/form-data} unless you override {@link #consumes()}.</li>
* <li>When the client supplies {@code ?async=true} the call is handed to
* {@link stirling.software.common.service.JobExecutorService JobExecutorService} where it may
* be queued, retried, tracked and subject to timeouts. For synchronous (default)
* invocations these advanced options are ignored.</li>
* <li>Progress information (see {@link #trackProgress()}) is stored in
* {@link stirling.software.common.service.TaskManager TaskManager} and can be
* polled via <code>GET /api/v1/general/job/{id}</code>.</li>
* </ul>
* </p>
* *
* <p>Unless stated otherwise an attribute only affects <em>async</em> execution.</p> * <p>Behaviour notes:
*
* <ul>
* <li>The endpoint is registered with {@code POST} and, by default, consumes {@code
* multipart/form-data} unless you override {@link #consumes()}.
* <li>When the client supplies {@code ?async=true} the call is handed to {@link
* stirling.software.common.service.JobExecutorService JobExecutorService} where it may be
* queued, retried, tracked and subject to timeouts. For synchronous (default) invocations
* these advanced options are ignored.
* <li>Progress information (see {@link #trackProgress()}) is stored in {@link
* stirling.software.common.service.TaskManager TaskManager} and can be polled via <code>
* GET /api/v1/general/job/{id}</code>.
* </ul>
*
* <p>Unless stated otherwise an attribute only affects <em>async</em> execution.
*/ */
@Target(ElementType.METHOD) @Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME) @Retention(RetentionPolicy.RUNTIME)
@ -31,42 +31,42 @@ import org.springframework.web.bind.annotation.RequestMethod;
@RequestMapping(method = RequestMethod.POST) @RequestMapping(method = RequestMethod.POST)
public @interface AutoJobPostMapping { public @interface AutoJobPostMapping {
/** /** Alias for {@link RequestMapping#value} the path mapping of the endpoint. */
* Alias for {@link RequestMapping#value} the path mapping of the endpoint.
*/
@AliasFor(annotation = RequestMapping.class, attribute = "value") @AliasFor(annotation = RequestMapping.class, attribute = "value")
String[] value() default {}; String[] value() default {};
/** /** MIME types this endpoint accepts. Defaults to {@code multipart/form-data}. */
* MIME types this endpoint accepts. Defaults to {@code multipart/form-data}.
*/
@AliasFor(annotation = RequestMapping.class, attribute = "consumes") @AliasFor(annotation = RequestMapping.class, attribute = "consumes")
String[] consumes() default {"multipart/form-data"}; String[] consumes() default {"multipart/form-data"};
/** /**
* Maximum execution time in milliseconds before the job is aborted. * Maximum execution time in milliseconds before the job is aborted. A negative value means "use
* A negative value means "use the application default". * the application default".
* <p>Only honoured when {@code async=true}.</p> *
* <p>Only honoured when {@code async=true}.
*/ */
long timeout() default -1; long timeout() default -1;
/** /**
* Total number of attempts (initial + retries). Must be at least&nbsp;1. * Total number of attempts (initial + retries). Must be at least&nbsp;1. Retries are executed
* Retries are executed with exponential backoff. * with exponential backoff.
* <p>Only honoured when {@code async=true}.</p> *
* <p>Only honoured when {@code async=true}.
*/ */
int retryCount() default 1; int retryCount() default 1;
/** /**
* Record percentage / note updates so they can be retrieved via the REST status endpoint. * Record percentage / note updates so they can be retrieved via the REST status endpoint.
* <p>Only honoured when {@code async=true}.</p> *
* <p>Only honoured when {@code async=true}.
*/ */
boolean trackProgress() default true; boolean trackProgress() default true;
/** /**
* If {@code true} the job may be placed in a queue instead of being rejected when resources * If {@code true} the job may be placed in a queue instead of being rejected when resources are
* are scarce. * scarce.
* <p>Only honoured when {@code async=true}.</p> *
* <p>Only honoured when {@code async=true}.
*/ */
boolean queueable() default false; boolean queueable() default false;

View File

@ -1,6 +1,5 @@
package stirling.software.common.config; package stirling.software.common.config;
import java.io.File;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
@ -14,7 +13,6 @@ import jakarta.annotation.PostConstruct;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import stirling.software.common.configuration.InstallationPathConfig;
import stirling.software.common.util.TempFileRegistry; import stirling.software.common.util.TempFileRegistry;
/** /**

View File

@ -173,7 +173,9 @@ public class ResourceMonitor {
log.info("System resource status changed from {} to {}", oldStatus, newStatus); log.info("System resource status changed from {} to {}", oldStatus, newStatus);
log.info( log.info(
"Current metrics - CPU: {}%, Memory: {}%, Free Memory: {} MB", "Current metrics - CPU: {}%, Memory: {}%, Free Memory: {} MB",
String.format("%.1f", cpuUsage * 100), String.format("%.1f", memoryUsage * 100), freeMemory / (1024 * 1024)); String.format("%.1f", cpuUsage * 100),
String.format("%.1f", memoryUsage * 100),
freeMemory / (1024 * 1024));
} }
} catch (Exception e) { } catch (Exception e) {
log.error("Error updating resource metrics: {}", e.getMessage(), e); log.error("Error updating resource metrics: {}", e.getMessage(), e);

View File

@ -57,31 +57,34 @@ public class TempFileCleanupService {
private static final int MAX_RECURSION_DEPTH = 5; private static final int MAX_RECURSION_DEPTH = 5;
// File patterns that identify our temp files // File patterns that identify our temp files
private static final Predicate<String> IS_OUR_TEMP_FILE = fileName -> private static final Predicate<String> IS_OUR_TEMP_FILE =
fileName.startsWith("stirling-pdf-") || fileName ->
fileName.startsWith("output_") || fileName.startsWith("stirling-pdf-")
fileName.startsWith("compressedPDF") || || fileName.startsWith("output_")
fileName.startsWith("pdf-save-") || || fileName.startsWith("compressedPDF")
fileName.startsWith("pdf-stream-") || || fileName.startsWith("pdf-save-")
fileName.startsWith("PDFBox") || || fileName.startsWith("pdf-stream-")
fileName.startsWith("input_") || || fileName.startsWith("PDFBox")
fileName.startsWith("overlay-"); || fileName.startsWith("input_")
|| fileName.startsWith("overlay-");
// File patterns that identify common system temp files // File patterns that identify common system temp files
private static final Predicate<String> IS_SYSTEM_TEMP_FILE = fileName -> private static final Predicate<String> IS_SYSTEM_TEMP_FILE =
fileName.matches("lu\\d+[a-z0-9]*\\.tmp") || fileName ->
fileName.matches("ocr_process\\d+") || fileName.matches("lu\\d+[a-z0-9]*\\.tmp")
(fileName.startsWith("tmp") && !fileName.contains("jetty")) || || fileName.matches("ocr_process\\d+")
fileName.startsWith("OSL_PIPE_") || || (fileName.startsWith("tmp") && !fileName.contains("jetty"))
(fileName.endsWith(".tmp") && !fileName.contains("jetty")); || fileName.startsWith("OSL_PIPE_")
|| (fileName.endsWith(".tmp") && !fileName.contains("jetty"));
// File patterns that should be excluded from cleanup // File patterns that should be excluded from cleanup
private static final Predicate<String> SHOULD_SKIP = fileName -> private static final Predicate<String> SHOULD_SKIP =
fileName.contains("jetty") || fileName ->
fileName.startsWith("jetty-") || fileName.contains("jetty")
fileName.equals("proc") || || fileName.startsWith("jetty-")
fileName.equals("sys") || || fileName.equals("proc")
fileName.equals("dev"); || fileName.equals("sys")
|| fileName.equals("dev");
@Autowired @Autowired
public TempFileCleanupService(TempFileRegistry registry, TempFileManager tempFileManager) { public TempFileCleanupService(TempFileRegistry registry, TempFileManager tempFileManager) {
@ -190,25 +193,28 @@ public class TempFileCleanupService {
* @param maxAgeMillis Maximum age of files to clean in milliseconds * @param maxAgeMillis Maximum age of files to clean in milliseconds
* @return Number of files deleted * @return Number of files deleted
*/ */
private int cleanupUnregisteredFiles(boolean containerMode, boolean isScheduled, long maxAgeMillis) { private int cleanupUnregisteredFiles(
boolean containerMode, boolean isScheduled, long maxAgeMillis) {
AtomicInteger totalDeletedCount = new AtomicInteger(0); AtomicInteger totalDeletedCount = new AtomicInteger(0);
try { try {
// Get all directories we need to clean // Get all directories we need to clean
Path systemTempPath = getSystemTempPath(); Path systemTempPath = getSystemTempPath();
Path[] dirsToScan = { Path[] dirsToScan = {
systemTempPath, systemTempPath, Path.of(customTempDirectory), Path.of(libreOfficeTempDir)
Path.of(customTempDirectory),
Path.of(libreOfficeTempDir)
}; };
// Process each directory // Process each directory
Arrays.stream(dirsToScan) Arrays.stream(dirsToScan)
.filter(Files::exists) .filter(Files::exists)
.forEach(tempDir -> { .forEach(
tempDir -> {
try { try {
String phase = isScheduled ? "scheduled" : "startup"; String phase = isScheduled ? "scheduled" : "startup";
log.info("Scanning directory for {} cleanup: {}", phase, tempDir); log.info(
"Scanning directory for {} cleanup: {}",
phase,
tempDir);
AtomicInteger dirDeletedCount = new AtomicInteger(0); AtomicInteger dirDeletedCount = new AtomicInteger(0);
cleanupDirectoryStreaming( cleanupDirectoryStreaming(
@ -220,15 +226,20 @@ public class TempFileCleanupService {
path -> { path -> {
dirDeletedCount.incrementAndGet(); dirDeletedCount.incrementAndGet();
if (log.isDebugEnabled()) { if (log.isDebugEnabled()) {
log.debug("Deleted temp file during {} cleanup: {}", phase, path); log.debug(
"Deleted temp file during {} cleanup: {}",
phase,
path);
} }
} });
);
int count = dirDeletedCount.get(); int count = dirDeletedCount.get();
totalDeletedCount.addAndGet(count); totalDeletedCount.addAndGet(count);
if (count > 0) { if (count > 0) {
log.info("Cleaned up {} files/directories in {}", count, tempDir); log.info(
"Cleaned up {} files/directories in {}",
count,
tempDir);
} }
} catch (IOException e) { } catch (IOException e) {
log.error("Error during cleanup of directory: {}", tempDir, e); log.error("Error during cleanup of directory: {}", tempDir, e);
@ -241,9 +252,7 @@ public class TempFileCleanupService {
return totalDeletedCount.get(); return totalDeletedCount.get();
} }
/** /** Get the system temp directory path based on configuration or system property. */
* Get the system temp directory path based on configuration or system property.
*/
private Path getSystemTempPath() { private Path getSystemTempPath() {
if (systemTempDir != null && !systemTempDir.isEmpty()) { if (systemTempDir != null && !systemTempDir.isEmpty()) {
return Path.of(systemTempDir); return Path.of(systemTempDir);
@ -252,9 +261,7 @@ public class TempFileCleanupService {
} }
} }
/** /** Determine if we're running in a container environment. */
* Determine if we're running in a container environment.
*/
private boolean isContainerMode() { private boolean isContainerMode() {
return "Docker".equals(machineType) || "Kubernetes".equals(machineType); return "Docker".equals(machineType) || "Kubernetes".equals(machineType);
} }
@ -276,7 +283,8 @@ public class TempFileCleanupService {
int depth, int depth,
long maxAgeMillis, long maxAgeMillis,
boolean isScheduled, boolean isScheduled,
Consumer<Path> onDeleteCallback) throws IOException { Consumer<Path> onDeleteCallback)
throws IOException {
// Check recursion depth limit // Check recursion depth limit
if (depth > MAX_RECURSION_DEPTH) { if (depth > MAX_RECURSION_DEPTH) {
@ -287,7 +295,8 @@ public class TempFileCleanupService {
// Use try-with-resources to ensure the stream is closed // Use try-with-resources to ensure the stream is closed
try (Stream<Path> pathStream = Files.list(directory)) { try (Stream<Path> pathStream = Files.list(directory)) {
// Process files in a streaming fashion instead of materializing the whole list // Process files in a streaming fashion instead of materializing the whole list
pathStream.forEach(path -> { pathStream.forEach(
path -> {
try { try {
String fileName = path.getFileName().toString(); String fileName = path.getFileName().toString();
@ -300,7 +309,12 @@ public class TempFileCleanupService {
if (Files.isDirectory(path)) { if (Files.isDirectory(path)) {
try { try {
cleanupDirectoryStreaming( cleanupDirectoryStreaming(
path, containerMode, depth + 1, maxAgeMillis, isScheduled, onDeleteCallback); path,
containerMode,
depth + 1,
maxAgeMillis,
isScheduled,
onDeleteCallback);
} catch (IOException e) { } catch (IOException e) {
log.warn("Error processing subdirectory: {}", path, e); log.warn("Error processing subdirectory: {}", path, e);
} }
@ -308,7 +322,7 @@ public class TempFileCleanupService {
} }
// Skip registered files - these are handled by TempFileManager // Skip registered files - these are handled by TempFileManager
if (isScheduled && registry.contains(path.toFile())) { if (registry.contains(path.toFile())) {
return; return;
} }
@ -319,7 +333,9 @@ public class TempFileCleanupService {
onDeleteCallback.accept(path); onDeleteCallback.accept(path);
} catch (IOException e) { } catch (IOException e) {
// Handle locked files more gracefully // Handle locked files more gracefully
if (e.getMessage() != null && e.getMessage().contains("being used by another process")) { if (e.getMessage() != null
&& e.getMessage()
.contains("being used by another process")) {
log.debug("File locked, skipping delete: {}", path); log.debug("File locked, skipping delete: {}", path);
} else { } else {
log.warn("Failed to delete temp file: {}", path, e); log.warn("Failed to delete temp file: {}", path, e);
@ -333,13 +349,14 @@ public class TempFileCleanupService {
} }
} }
/** /** Determine if a file should be deleted based on its name, age, and other criteria. */
* Determine if a file should be deleted based on its name, age, and other criteria. private boolean shouldDeleteFile(
*/ Path path, String fileName, boolean containerMode, long maxAgeMillis) {
private boolean shouldDeleteFile(Path path, String fileName, boolean containerMode, long maxAgeMillis) {
// First check if it matches our known temp file patterns // First check if it matches our known temp file patterns
boolean isOurTempFile = IS_OUR_TEMP_FILE.test(fileName); boolean isOurTempFile = IS_OUR_TEMP_FILE.test(fileName);
boolean isSystemTempFile = IS_SYSTEM_TEMP_FILE.test(fileName); boolean isSystemTempFile = IS_SYSTEM_TEMP_FILE.test(fileName);
// Normal operation - check against temp file patterns
boolean shouldDelete = isOurTempFile || (containerMode && isSystemTempFile); boolean shouldDelete = isOurTempFile || (containerMode && isSystemTempFile);
// Get file info for age checks // Get file info for age checks
@ -362,8 +379,10 @@ public class TempFileCleanupService {
log.debug("Could not check file info, skipping: {}", path); log.debug("Could not check file info, skipping: {}", path);
} }
// Check file age against maxAgeMillis only if it's not an empty file that we've already decided to delete // Check file age against maxAgeMillis only if it's not an empty file that we've already
// decided to delete
if (!isEmptyFile && shouldDelete && maxAgeMillis > 0) { if (!isEmptyFile && shouldDelete && maxAgeMillis > 0) {
// In normal mode, check age against maxAgeMillis
shouldDelete = (currentTime - lastModified) > maxAgeMillis; shouldDelete = (currentTime - lastModified) > maxAgeMillis;
} }
@ -385,8 +404,7 @@ public class TempFileCleanupService {
0, 0,
0, // age doesn't matter for LibreOffice cleanup 0, // age doesn't matter for LibreOffice cleanup
false, false,
path -> log.debug("Cleaned up LibreOffice temp file: {}", path) path -> log.debug("Cleaned up LibreOffice temp file: {}", path));
);
log.debug("Cleaned up LibreOffice temp directory contents: {}", dir); log.debug("Cleaned up LibreOffice temp directory contents: {}", dir);
} }
} }

View File

@ -37,7 +37,6 @@ public class TempFileManager {
@Value("${stirling.tempfiles.max-age-hours:24}") @Value("${stirling.tempfiles.max-age-hours:24}")
private long maxAgeHours; private long maxAgeHours;
@Autowired @Autowired
public TempFileManager(TempFileRegistry registry) { public TempFileManager(TempFileRegistry registry) {
this.registry = registry; this.registry = registry;

View File

@ -10,19 +10,19 @@ import java.io.IOException;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.attribute.FileTime; import java.nio.file.attribute.FileTime;
import java.util.Arrays;
import java.util.HashSet; import java.util.HashSet;
import java.util.Set; import java.util.Set;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer; import java.util.function.Consumer;
import java.util.stream.Stream;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.api.io.TempDir;
import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks; import org.mockito.InjectMocks;
import org.mockito.Mock; import org.mockito.Mock;
import org.mockito.MockedStatic;
import org.mockito.MockitoAnnotations; import org.mockito.MockitoAnnotations;
import org.springframework.test.util.ReflectionTestUtils; import org.springframework.test.util.ReflectionTestUtils;
@ -67,7 +67,7 @@ public class TempFileCleanupServiceTest {
ReflectionTestUtils.setField(cleanupService, "systemTempDir", systemTempDir.toString()); ReflectionTestUtils.setField(cleanupService, "systemTempDir", systemTempDir.toString());
ReflectionTestUtils.setField(cleanupService, "customTempDirectory", customTempDir.toString()); ReflectionTestUtils.setField(cleanupService, "customTempDirectory", customTempDir.toString());
ReflectionTestUtils.setField(cleanupService, "libreOfficeTempDir", libreOfficeTempDir.toString()); ReflectionTestUtils.setField(cleanupService, "libreOfficeTempDir", libreOfficeTempDir.toString());
ReflectionTestUtils.setField(cleanupService, "machineType", "Docker"); // Test in container mode ReflectionTestUtils.setField(cleanupService, "machineType", "Standard"); // Regular mode
ReflectionTestUtils.setField(cleanupService, "performStartupCleanup", false); // Disable auto-startup cleanup ReflectionTestUtils.setField(cleanupService, "performStartupCleanup", false); // Disable auto-startup cleanup
when(tempFileManager.getMaxAgeMillis()).thenReturn(3600000L); // 1 hour when(tempFileManager.getMaxAgeMillis()).thenReturn(3600000L); // 1 hour
@ -98,6 +98,9 @@ public class TempFileCleanupServiceTest {
Path ourTempFile4 = Files.createFile(customTempDir.resolve("pdf-save-123-456.tmp")); Path ourTempFile4 = Files.createFile(customTempDir.resolve("pdf-save-123-456.tmp"));
Path ourTempFile5 = Files.createFile(libreOfficeTempDir.resolve("input_file.pdf")); Path ourTempFile5 = Files.createFile(libreOfficeTempDir.resolve("input_file.pdf"));
// Old temporary files
Path oldTempFile = Files.createFile(systemTempDir.resolve("output_old.pdf"));
// System temp files that should be cleaned in container mode // System temp files that should be cleaned in container mode
Path sysTempFile1 = Files.createFile(systemTempDir.resolve("lu123abc.tmp")); Path sysTempFile1 = Files.createFile(systemTempDir.resolve("lu123abc.tmp"));
Path sysTempFile2 = Files.createFile(customTempDir.resolve("ocr_process123")); Path sysTempFile2 = Files.createFile(customTempDir.resolve("ocr_process123"));
@ -118,45 +121,217 @@ public class TempFileCleanupServiceTest {
// Configure mock registry to say these files aren't registered // Configure mock registry to say these files aren't registered
when(registry.contains(any(File.class))).thenReturn(false); when(registry.contains(any(File.class))).thenReturn(false);
// Create a file older than threshold // The set of files that will be deleted in our test
Path oldFile = Files.createFile(systemTempDir.resolve("output_old.pdf")); Set<Path> deletedFiles = new HashSet<>();
Files.setLastModifiedTime(oldFile, FileTime.from(
Files.getLastModifiedTime(oldFile).toMillis() - 5000000,
TimeUnit.MILLISECONDS));
// Act // Use MockedStatic to mock Files operations
invokeCleanupDirectoryStreaming(systemTempDir, true, 0, 3600000); try (MockedStatic<Files> mockedFiles = mockStatic(Files.class)) {
invokeCleanupDirectoryStreaming(customTempDir, true, 0, 3600000); // Mock Files.list for each directory we'll process
invokeCleanupDirectoryStreaming(libreOfficeTempDir, true, 0, 3600000); mockedFiles.when(() -> Files.list(eq(systemTempDir)))
.thenReturn(Stream.of(
ourTempFile1, ourTempFile2, oldTempFile, sysTempFile1,
jettyFile1, jettyFile2, regularFile, emptyFile, nestedDir));
// Assert - Our temp files and system temp files should be deleted (if old enough) mockedFiles.when(() -> Files.list(eq(customTempDir)))
assertFalse(Files.exists(oldFile), "Old temp file should be deleted"); .thenReturn(Stream.of(ourTempFile3, ourTempFile4, sysTempFile2, sysTempFile3));
assertTrue(Files.exists(ourTempFile1), "Recent temp file should be preserved");
assertTrue(Files.exists(sysTempFile1), "Recent system temp file should be preserved"); mockedFiles.when(() -> Files.list(eq(libreOfficeTempDir)))
.thenReturn(Stream.of(ourTempFile5));
mockedFiles.when(() -> Files.list(eq(nestedDir)))
.thenReturn(Stream.of(nestedTempFile));
// Configure Files.isDirectory for each path
mockedFiles.when(() -> Files.isDirectory(eq(nestedDir))).thenReturn(true);
mockedFiles.when(() -> Files.isDirectory(any(Path.class))).thenReturn(false);
// Configure Files.exists to return true for all paths
mockedFiles.when(() -> Files.exists(any(Path.class))).thenReturn(true);
// Configure Files.getLastModifiedTime to return different times based on file names
mockedFiles.when(() -> Files.getLastModifiedTime(any(Path.class)))
.thenAnswer(invocation -> {
Path path = invocation.getArgument(0);
String fileName = path.getFileName().toString();
// For files with "old" in the name, return a timestamp older than maxAgeMillis
if (fileName.contains("old")) {
return FileTime.fromMillis(System.currentTimeMillis() - 5000000);
}
// For empty.tmp file, return a timestamp older than 5 minutes (for empty file test)
else if (fileName.equals("empty.tmp")) {
return FileTime.fromMillis(System.currentTimeMillis() - 6 * 60 * 1000);
}
// For all other files, return a recent timestamp
else {
return FileTime.fromMillis(System.currentTimeMillis() - 60000); // 1 minute ago
}
});
// Configure Files.size to return different sizes based on file names
mockedFiles.when(() -> Files.size(any(Path.class)))
.thenAnswer(invocation -> {
Path path = invocation.getArgument(0);
String fileName = path.getFileName().toString();
// Return 0 bytes for the empty file
if (fileName.equals("empty.tmp")) {
return 0L;
}
// Return normal size for all other files
else {
return 1024L; // 1 KB
}
});
// For deleteIfExists, track which files would be deleted
mockedFiles.when(() -> Files.deleteIfExists(any(Path.class)))
.thenAnswer(invocation -> {
Path path = invocation.getArgument(0);
deletedFiles.add(path);
return true;
});
// Act - set containerMode to false for this test
invokeCleanupDirectoryStreaming(systemTempDir, false, 0, 3600000);
invokeCleanupDirectoryStreaming(customTempDir, false, 0, 3600000);
invokeCleanupDirectoryStreaming(libreOfficeTempDir, false, 0, 3600000);
// Assert - Only old temp files and empty files should be deleted
assertTrue(deletedFiles.contains(oldTempFile), "Old temp file should be deleted");
assertTrue(deletedFiles.contains(emptyFile), "Empty file should be deleted");
// Regular temp files should not be deleted because they're too new
assertFalse(deletedFiles.contains(ourTempFile1), "Recent temp file should be preserved");
assertFalse(deletedFiles.contains(ourTempFile2), "Recent temp file should be preserved");
assertFalse(deletedFiles.contains(ourTempFile3), "Recent temp file should be preserved");
assertFalse(deletedFiles.contains(ourTempFile4), "Recent temp file should be preserved");
assertFalse(deletedFiles.contains(ourTempFile5), "Recent temp file should be preserved");
// System temp files should not be deleted in non-container mode
assertFalse(deletedFiles.contains(sysTempFile1), "System temp file should be preserved in non-container mode");
assertFalse(deletedFiles.contains(sysTempFile2), "System temp file should be preserved in non-container mode");
assertFalse(deletedFiles.contains(sysTempFile3), "System temp file should be preserved in non-container mode");
// Jetty files and regular files should never be deleted // Jetty files and regular files should never be deleted
assertTrue(Files.exists(jettyFile1), "Jetty file should be preserved"); assertFalse(deletedFiles.contains(jettyFile1), "Jetty file should be preserved");
assertTrue(Files.exists(jettyFile2), "File with jetty in name should be preserved"); assertFalse(deletedFiles.contains(jettyFile2), "File with jetty in name should be preserved");
assertTrue(Files.exists(regularFile), "Regular file should be preserved"); assertFalse(deletedFiles.contains(regularFile), "Regular file should be preserved");
}
}
@Test
public void testContainerModeCleanup() throws IOException {
// Arrange - Create various temp files
Path ourTempFile = Files.createFile(systemTempDir.resolve("output_123.pdf"));
Path sysTempFile = Files.createFile(systemTempDir.resolve("lu123abc.tmp"));
Path regularFile = Files.createFile(systemTempDir.resolve("important.txt"));
// Configure mock registry to say these files aren't registered
when(registry.contains(any(File.class))).thenReturn(false);
// The set of files that will be deleted in our test
Set<Path> deletedFiles = new HashSet<>();
// Use MockedStatic to mock Files operations
try (MockedStatic<Files> mockedFiles = mockStatic(Files.class)) {
// Mock Files.list for systemTempDir
mockedFiles.when(() -> Files.list(eq(systemTempDir)))
.thenReturn(Stream.of(ourTempFile, sysTempFile, regularFile));
// Configure Files.isDirectory
mockedFiles.when(() -> Files.isDirectory(any(Path.class))).thenReturn(false);
// Configure Files.exists
mockedFiles.when(() -> Files.exists(any(Path.class))).thenReturn(true);
// Configure Files.getLastModifiedTime to return recent timestamps
mockedFiles.when(() -> Files.getLastModifiedTime(any(Path.class)))
.thenReturn(FileTime.fromMillis(System.currentTimeMillis() - 60000)); // 1 minute ago
// Configure Files.size to return normal size
mockedFiles.when(() -> Files.size(any(Path.class)))
.thenReturn(1024L); // 1 KB
// For deleteIfExists, track which files would be deleted
mockedFiles.when(() -> Files.deleteIfExists(any(Path.class)))
.thenAnswer(invocation -> {
Path path = invocation.getArgument(0);
deletedFiles.add(path);
return true;
});
// Act - set containerMode to true and maxAgeMillis to 0 for container startup cleanup
invokeCleanupDirectoryStreaming(systemTempDir, true, 0, 0);
// Assert - In container mode, both our temp files and system temp files should be deleted
// regardless of age (when maxAgeMillis is 0)
assertTrue(deletedFiles.contains(ourTempFile), "Our temp file should be deleted in container mode");
assertTrue(deletedFiles.contains(sysTempFile), "System temp file should be deleted in container mode");
assertFalse(deletedFiles.contains(regularFile), "Regular file should be preserved");
}
} }
@Test @Test
public void testEmptyFileHandling() throws IOException { public void testEmptyFileHandling() throws IOException {
// Arrange - Create an empty file // Arrange - Create an empty file
Path emptyFile = Files.createFile(systemTempDir.resolve("empty.tmp")); Path emptyFile = Files.createFile(systemTempDir.resolve("empty.tmp"));
// Make it "old enough" to be deleted (>5 minutes) Path recentEmptyFile = Files.createFile(systemTempDir.resolve("recent_empty.tmp"));
Files.setLastModifiedTime(emptyFile, FileTime.from(
Files.getLastModifiedTime(emptyFile).toMillis() - 6 * 60 * 1000,
TimeUnit.MILLISECONDS));
// Configure mock registry to say this file isn't registered // Configure mock registry to say these files aren't registered
when(registry.contains(any(File.class))).thenReturn(false); when(registry.contains(any(File.class))).thenReturn(false);
// The set of files that will be deleted in our test
Set<Path> deletedFiles = new HashSet<>();
// Use MockedStatic to mock Files operations
try (MockedStatic<Files> mockedFiles = mockStatic(Files.class)) {
// Mock Files.list for systemTempDir
mockedFiles.when(() -> Files.list(eq(systemTempDir)))
.thenReturn(Stream.of(emptyFile, recentEmptyFile));
// Configure Files.isDirectory
mockedFiles.when(() -> Files.isDirectory(any(Path.class))).thenReturn(false);
// Configure Files.exists
mockedFiles.when(() -> Files.exists(any(Path.class))).thenReturn(true);
// Configure Files.getLastModifiedTime to return different times based on file names
mockedFiles.when(() -> Files.getLastModifiedTime(any(Path.class)))
.thenAnswer(invocation -> {
Path path = invocation.getArgument(0);
String fileName = path.getFileName().toString();
if (fileName.equals("empty.tmp")) {
// More than 5 minutes old
return FileTime.fromMillis(System.currentTimeMillis() - 6 * 60 * 1000);
} else {
// Less than 5 minutes old
return FileTime.fromMillis(System.currentTimeMillis() - 2 * 60 * 1000);
}
});
// Configure Files.size to return 0 for empty files
mockedFiles.when(() -> Files.size(any(Path.class)))
.thenReturn(0L);
// For deleteIfExists, track which files would be deleted
mockedFiles.when(() -> Files.deleteIfExists(any(Path.class)))
.thenAnswer(invocation -> {
Path path = invocation.getArgument(0);
deletedFiles.add(path);
return true;
});
// Act // Act
invokeCleanupDirectoryStreaming(systemTempDir, true, 0, 3600000); invokeCleanupDirectoryStreaming(systemTempDir, false, 0, 3600000);
// Assert // Assert
assertFalse(Files.exists(emptyFile), "Empty file older than 5 minutes should be deleted"); assertTrue(deletedFiles.contains(emptyFile),
"Empty file older than 5 minutes should be deleted");
assertFalse(deletedFiles.contains(recentEmptyFile),
"Empty file newer than 5 minutes should not be deleted");
}
} }
@Test @Test
@ -168,23 +343,79 @@ public class TempFileCleanupServiceTest {
Path tempFile1 = Files.createFile(dir1.resolve("output_1.pdf")); Path tempFile1 = Files.createFile(dir1.resolve("output_1.pdf"));
Path tempFile2 = Files.createFile(dir2.resolve("output_2.pdf")); Path tempFile2 = Files.createFile(dir2.resolve("output_2.pdf"));
Path tempFile3 = Files.createFile(dir3.resolve("output_3.pdf")); Path tempFile3 = Files.createFile(dir3.resolve("output_old_3.pdf"));
// Make the deepest file old enough to be deleted
Files.setLastModifiedTime(tempFile3, FileTime.from(
Files.getLastModifiedTime(tempFile3).toMillis() - 5000000,
TimeUnit.MILLISECONDS));
// Configure mock registry to say these files aren't registered // Configure mock registry to say these files aren't registered
when(registry.contains(any(File.class))).thenReturn(false); when(registry.contains(any(File.class))).thenReturn(false);
// The set of files that will be deleted in our test
Set<Path> deletedFiles = new HashSet<>();
// Use MockedStatic to mock Files operations
try (MockedStatic<Files> mockedFiles = mockStatic(Files.class)) {
// Mock Files.list for each directory
mockedFiles.when(() -> Files.list(eq(systemTempDir)))
.thenReturn(Stream.of(dir1));
mockedFiles.when(() -> Files.list(eq(dir1)))
.thenReturn(Stream.of(tempFile1, dir2));
mockedFiles.when(() -> Files.list(eq(dir2)))
.thenReturn(Stream.of(tempFile2, dir3));
mockedFiles.when(() -> Files.list(eq(dir3)))
.thenReturn(Stream.of(tempFile3));
// Configure Files.isDirectory for each path
mockedFiles.when(() -> Files.isDirectory(eq(dir1))).thenReturn(true);
mockedFiles.when(() -> Files.isDirectory(eq(dir2))).thenReturn(true);
mockedFiles.when(() -> Files.isDirectory(eq(dir3))).thenReturn(true);
mockedFiles.when(() -> Files.isDirectory(eq(tempFile1))).thenReturn(false);
mockedFiles.when(() -> Files.isDirectory(eq(tempFile2))).thenReturn(false);
mockedFiles.when(() -> Files.isDirectory(eq(tempFile3))).thenReturn(false);
// Configure Files.exists to return true for all paths
mockedFiles.when(() -> Files.exists(any(Path.class))).thenReturn(true);
// Configure Files.getLastModifiedTime to return different times based on file names
mockedFiles.when(() -> Files.getLastModifiedTime(any(Path.class)))
.thenAnswer(invocation -> {
Path path = invocation.getArgument(0);
String fileName = path.getFileName().toString();
if (fileName.contains("old")) {
// Old file
return FileTime.fromMillis(System.currentTimeMillis() - 5000000);
} else {
// Recent file
return FileTime.fromMillis(System.currentTimeMillis() - 60000);
}
});
// Configure Files.size to return normal size
mockedFiles.when(() -> Files.size(any(Path.class)))
.thenReturn(1024L);
// For deleteIfExists, track which files would be deleted
mockedFiles.when(() -> Files.deleteIfExists(any(Path.class)))
.thenAnswer(invocation -> {
Path path = invocation.getArgument(0);
deletedFiles.add(path);
return true;
});
// Act // Act
invokeCleanupDirectoryStreaming(systemTempDir, true, 0, 3600000); invokeCleanupDirectoryStreaming(systemTempDir, false, 0, 3600000);
// Debug - print what was deleted
System.out.println("Deleted files: " + deletedFiles);
System.out.println("Looking for: " + tempFile3);
// Assert // Assert
assertTrue(Files.exists(tempFile1), "Recent temp file should be preserved"); assertFalse(deletedFiles.contains(tempFile1), "Recent temp file should be preserved");
assertTrue(Files.exists(tempFile2), "Recent temp file should be preserved"); assertFalse(deletedFiles.contains(tempFile2), "Recent temp file should be preserved");
assertFalse(Files.exists(tempFile3), "Old temp file in nested directory should be deleted"); assertTrue(deletedFiles.contains(tempFile3), "Old temp file in nested directory should be deleted");
}
} }
/** /**
@ -197,7 +428,7 @@ public class TempFileCleanupServiceTest {
AtomicInteger deleteCount = new AtomicInteger(0); AtomicInteger deleteCount = new AtomicInteger(0);
Consumer<Path> deleteCallback = path -> deleteCount.incrementAndGet(); Consumer<Path> deleteCallback = path -> deleteCount.incrementAndGet();
// Get the new method with updated signature // Get the method with updated signature
var method = TempFileCleanupService.class.getDeclaredMethod( var method = TempFileCleanupService.class.getDeclaredMethod(
"cleanupDirectoryStreaming", "cleanupDirectoryStreaming",
Path.class, boolean.class, int.class, long.class, boolean.class, Consumer.class); Path.class, boolean.class, int.class, long.class, boolean.class, Consumer.class);
@ -209,4 +440,9 @@ public class TempFileCleanupServiceTest {
throw new RuntimeException("Error invoking cleanupDirectoryStreaming", e); throw new RuntimeException("Error invoking cleanupDirectoryStreaming", e);
} }
} }
// Matcher for exact path equality
private static Path eq(Path path) {
return argThat(arg -> arg != null && arg.equals(path));
}
} }

View File

@ -11,7 +11,9 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import io.github.pixee.security.SystemCommand; import io.github.pixee.security.SystemCommand;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import stirling.software.common.service.TempFileCleanupService; import stirling.software.common.service.TempFileCleanupService;
import stirling.software.common.util.ApplicationContextProvider; import stirling.software.common.util.ApplicationContextProvider;
import stirling.software.common.util.TempFileManager; import stirling.software.common.util.TempFileManager;
@ -143,9 +145,7 @@ public class UnoconvServer {
} }
} }
/** /** Notify that unoconv is being used, to reset the inactivity timer. */
* Notify that unoconv is being used, to reset the inactivity timer.
*/
public void notifyActivity() { public void notifyActivity() {
lastActivityTime = System.currentTimeMillis(); lastActivityTime = System.currentTimeMillis();
} }

View File

@ -10,6 +10,7 @@ import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.ApplicationProperties; import stirling.software.common.model.ApplicationProperties;
@Service @Service

View File

@ -35,8 +35,6 @@ import stirling.software.common.service.CustomPDFDocumentFactory;
import stirling.software.common.util.ProcessExecutor; import stirling.software.common.util.ProcessExecutor;
import stirling.software.common.util.ProcessExecutor.ProcessExecutorResult; import stirling.software.common.util.ProcessExecutor.ProcessExecutorResult;
import stirling.software.common.util.TempFileManager; import stirling.software.common.util.TempFileManager;
import stirling.software.common.util.TempFileUtil;
import stirling.software.common.util.TempFileUtil.TempFile;
@RestController @RestController
@RequestMapping("/api/v1/misc") @RequestMapping("/api/v1/misc")
@ -112,18 +110,21 @@ public class OCRController {
hasText = !stripper.getText(tempDoc).trim().isEmpty(); hasText = !stripper.getText(tempDoc).trim().isEmpty();
} }
boolean shouldOcr = switch (ocrType) { boolean shouldOcr =
switch (ocrType) {
case "skip-text" -> !hasText; case "skip-text" -> !hasText;
case "force-ocr" -> true; case "force-ocr" -> true;
default -> true; default -> true;
}; };
File pageOutputPath = new File(tempOutputDir, String.format("page_%d.pdf", pageNum)); File pageOutputPath =
new File(tempOutputDir, String.format("page_%d.pdf", pageNum));
if (shouldOcr) { if (shouldOcr) {
// Convert page to image // Convert page to image
BufferedImage image = pdfRenderer.renderImageWithDPI(pageNum, 300); BufferedImage image = pdfRenderer.renderImageWithDPI(pageNum, 300);
File imagePath = new File(tempImagesDir, String.format("page_%d.png", pageNum)); File imagePath =
new File(tempImagesDir, String.format("page_%d.png", pageNum));
ImageIO.write(image, "png", imagePath); ImageIO.write(image, "png", imagePath);
// Build OCR command // Build OCR command
@ -140,16 +141,22 @@ public class OCRController {
// Use ProcessExecutor to run tesseract command // Use ProcessExecutor to run tesseract command
try { try {
ProcessExecutorResult result = ProcessExecutor.getInstance(ProcessExecutor.Processes.TESSERACT) ProcessExecutorResult result =
ProcessExecutor.getInstance(ProcessExecutor.Processes.TESSERACT)
.runCommandWithOutputHandling(command); .runCommandWithOutputHandling(command);
log.debug("Tesseract OCR completed for page {} with exit code {}", log.debug(
pageNum, result.getRc()); "Tesseract OCR completed for page {} with exit code {}",
pageNum,
result.getRc());
// Add OCR'd PDF to merger // Add OCR'd PDF to merger
merger.addSource(pageOutputPath); merger.addSource(pageOutputPath);
} catch (IOException | InterruptedException e) { } catch (IOException | InterruptedException e) {
log.error("Error processing page {} with tesseract: {}", pageNum, e.getMessage()); log.error(
"Error processing page {} with tesseract: {}",
pageNum,
e.getMessage());
// If OCR fails, fall back to the original page // If OCR fails, fall back to the original page
try (PDDocument pageDoc = new PDDocument()) { try (PDDocument pageDoc = new PDDocument()) {
pageDoc.addPage(page); pageDoc.addPage(page);

View File

@ -1,8 +1,6 @@
package stirling.software.SPDF.controller.api.misc; package stirling.software.SPDF.controller.api.misc;
import java.io.IOException; import java.io.IOException;
import stirling.software.common.util.TempFileManager;
import stirling.software.common.util.TempFileUtil;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@ -23,6 +21,8 @@ import stirling.software.common.model.api.PDFFile;
import stirling.software.common.service.CustomPDFDocumentFactory; import stirling.software.common.service.CustomPDFDocumentFactory;
import stirling.software.common.util.ProcessExecutor; import stirling.software.common.util.ProcessExecutor;
import stirling.software.common.util.ProcessExecutor.ProcessExecutorResult; import stirling.software.common.util.ProcessExecutor.ProcessExecutorResult;
import stirling.software.common.util.TempFileManager;
import stirling.software.common.util.TempFileUtil;
import stirling.software.common.util.WebResponseUtils; import stirling.software.common.util.WebResponseUtils;
@RestController @RestController

View File

@ -6,8 +6,6 @@ import java.io.File;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import stirling.software.common.util.TempFileManager;
import stirling.software.common.util.TempFileUtil;
import java.util.List; import java.util.List;
import javax.imageio.ImageIO; import javax.imageio.ImageIO;
@ -41,6 +39,8 @@ import lombok.RequiredArgsConstructor;
import stirling.software.SPDF.model.api.misc.AddStampRequest; import stirling.software.SPDF.model.api.misc.AddStampRequest;
import stirling.software.common.service.CustomPDFDocumentFactory; import stirling.software.common.service.CustomPDFDocumentFactory;
import stirling.software.common.util.TempFileManager;
import stirling.software.common.util.TempFileUtil;
import stirling.software.common.util.WebResponseUtils; import stirling.software.common.util.WebResponseUtils;
@RestController @RestController
@ -52,8 +52,6 @@ public class StampController {
private final CustomPDFDocumentFactory pdfDocumentFactory; private final CustomPDFDocumentFactory pdfDocumentFactory;
private final TempFileManager tempFileManager; private final TempFileManager tempFileManager;
@PostMapping(consumes = "multipart/form-data", value = "/add-stamp") @PostMapping(consumes = "multipart/form-data", value = "/add-stamp")
@Operation( @Operation(
summary = "Add stamp to a PDF file", summary = "Add stamp to a PDF file",
@ -194,7 +192,8 @@ public class StampController {
String fileExtension = resourceDir.substring(resourceDir.lastIndexOf(".")); String fileExtension = resourceDir.substring(resourceDir.lastIndexOf("."));
// Use TempFileUtil.TempFile with try-with-resources for automatic cleanup // Use TempFileUtil.TempFile with try-with-resources for automatic cleanup
try (TempFileUtil.TempFile tempFileWrapper = new TempFileUtil.TempFile(tempFileManager, fileExtension)) { try (TempFileUtil.TempFile tempFileWrapper =
new TempFileUtil.TempFile(tempFileManager, fileExtension)) {
File tempFile = tempFileWrapper.getFile(); File tempFile = tempFileWrapper.getFile();
try (InputStream is = classPathResource.getInputStream(); try (InputStream is = classPathResource.getInputStream();
FileOutputStream os = new FileOutputStream(tempFile)) { FileOutputStream os = new FileOutputStream(tempFile)) {