Mirrors/Stirling-PDF
Mirrors/Stirling-PDF
1
0
Fork 0
mirror of https://github.com/Frooodle/Stirling-PDF.git synced 2024-12-21 19:08:24 +01:00
Code Issues Packages Projects Releases Wiki Activity

backup

Browse Source
This commit is contained in:
Anthony Stirling 2024-11-28 15:00:37 +00:00
parent d832a90de0
commit 2a4a19a80f
10 changed files with 235 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
cucumber/features/external.feature
View File

@ -48,24 +48,6 @@ Feature: API Validation
And the response status code should be 200
@ocr @negative
Scenario: Process PDF with text and OCR with type normal
Given I generate a PDF file as "fileInput"
And the pdf contains 3 pages with random text
And the request data includes
| parameter | value |
| languages | eng |
| sidecar | false |
| deskew | true |
| clean | true |
| cleanFinal | true |
| ocrType | Normal |
| ocrRenderType | hocr |
| removeImagesAfter| false |
When I send the API request to the endpoint "/api/v1/misc/ocr-pdf"
Then the response status code should be 500
@ocr @positive
Scenario: Process PDF with OCR
Given I generate a PDF file as "fileInput"
@ -84,26 +66,6 @@ Feature: API Validation
And the response file should have size greater than 0
And the response status code should be 200
@ocr @positive
Scenario: Process PDF with OCR with sidecar
Given I generate a PDF file as "fileInput"
And the request data includes
| parameter | value |
| languages | eng |
| sidecar | true |
| deskew | true |
| clean | true |
| cleanFinal | true |
| ocrType | Force |
| ocrRenderType | hocr |
| removeImagesAfter| false |
When I send the API request to the endpoint "/api/v1/misc/ocr-pdf"
Then the response content type should be "application/octet-stream"
And the response file should have extension ".zip"
And the response ZIP should contain 2 files
And the response file should have size greater than 0
And the response status code should be 200
@libre @positive
Scenario Outline: Convert PDF to various word formats

1
src/main/java/stirling/software/SPDF/config/ExternalAppDepConfig.java
View File

@ -43,7 +43,6 @@ public class ExternalAppDepConfig {
put("unoconv", List.of("Unoconv"));
put("qpdf", List.of("qpdf"));
put("tesseract", List.of("tesseract"));
}
};

155
src/main/java/stirling/software/SPDF/config/security/SecurityConfiguration.java
View File

@ -1,8 +1,12 @@
package stirling.software.SPDF.config.security;
import java.io.BufferedReader;
import java.io.IOException;
import java.security.cert.X509Certificate;
import java.time.Duration;
import java.util.*;
import org.opensaml.saml.common.assertion.ValidationContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
@ -11,6 +15,7 @@ import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Lazy;
import org.springframework.core.io.Resource;
import org.springframework.security.authentication.AuthenticationProvider;
import org.springframework.security.authentication.ProviderManager;
import org.springframework.security.authentication.dao.DaoAuthenticationProvider;
import org.springframework.security.config.annotation.method.configuration.EnableMethodSecurity;
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
@ -26,21 +31,33 @@ import org.springframework.security.oauth2.client.registration.ClientRegistratio
import org.springframework.security.oauth2.client.registration.ClientRegistrations;
import org.springframework.security.oauth2.client.registration.InMemoryClientRegistrationRepository;
import org.springframework.security.oauth2.core.user.OAuth2UserAuthority;
import org.springframework.security.saml2.core.Saml2ErrorCodes;
import org.springframework.security.saml2.core.Saml2ResponseValidatorResult;
import org.springframework.security.saml2.core.Saml2X509Credential;
import org.springframework.security.saml2.core.Saml2X509Credential.Saml2X509CredentialType;
import org.springframework.security.saml2.provider.service.authentication.OpenSaml4AuthenticationProvider;
import org.springframework.security.saml2.provider.service.registration.InMemoryRelyingPartyRegistrationRepository;
import org.springframework.security.saml2.provider.service.registration.RelyingPartyRegistration;
import org.springframework.security.saml2.provider.service.registration.RelyingPartyRegistrationRepository;
import org.springframework.security.saml2.provider.service.registration.Saml2MessageBinding;
import org.springframework.security.saml2.provider.service.web.HttpSessionSaml2AuthenticationRequestRepository;
import org.springframework.security.saml2.provider.service.web.Saml2WebSsoAuthenticationRequestFilter;
import org.springframework.security.saml2.provider.service.web.authentication.OpenSaml4AuthenticationRequestResolver;
import org.springframework.security.saml2.provider.service.web.authentication.Saml2WebSsoAuthenticationFilter;
import org.springframework.security.web.SecurityFilterChain;
import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter;
import org.springframework.security.web.authentication.rememberme.PersistentTokenRepository;
import org.springframework.security.web.context.SecurityContextHolderFilter;
import org.springframework.security.web.csrf.CookieCsrfTokenRepository;
import org.springframework.security.web.csrf.CsrfTokenRequestAttributeHandler;
import org.springframework.security.web.savedrequest.NullRequestCache;
import org.springframework.security.web.util.matcher.AntPathRequestMatcher;
import org.springframework.web.filter.OncePerRequestFilter;
import jakarta.servlet.FilterChain;
import jakarta.servlet.ServletException;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import lombok.extern.slf4j.Slf4j;
import stirling.software.SPDF.config.security.oauth2.CustomOAuth2AuthenticationFailureHandler;
import stirling.software.SPDF.config.security.oauth2.CustomOAuth2AuthenticationSuccessHandler;
@ -49,6 +66,7 @@ import stirling.software.SPDF.config.security.saml2.CertificateUtils;
import stirling.software.SPDF.config.security.saml2.CustomSaml2AuthenticationFailureHandler;
import stirling.software.SPDF.config.security.saml2.CustomSaml2AuthenticationSuccessHandler;
import stirling.software.SPDF.config.security.saml2.CustomSaml2ResponseAuthenticationConverter;
import stirling.software.SPDF.config.security.saml2.SamlDebugFilter;
import stirling.software.SPDF.config.security.session.SessionPersistentRegistry;
import stirling.software.SPDF.model.ApplicationProperties;
import stirling.software.SPDF.model.ApplicationProperties.Security.OAUTH2;
@ -87,7 +105,8 @@ public class SecurityConfiguration {
@Autowired private FirstLoginFilter firstLoginFilter;
@Autowired private SessionPersistentRegistry sessionRegistry;
@Autowired
private SamlDebugFilter samlDebugFilter;
@Bean
public SecurityFilterChain filterChain(HttpSecurity http) throws Exception {
@ -245,12 +264,17 @@ public class SecurityConfiguration {
}
// Handle SAML
if (applicationProperties.getSecurity().isSaml2Activ()
&& applicationProperties.getSystem().getEnableAlphaFunctionality()) {
http.authenticationProvider(samlAuthenticationProvider());
http.saml2Login(
saml2 ->
saml2.loginPage("/saml2")
if (applicationProperties.getSecurity().isSaml2Activ()) {
http.authenticationProvider(samlAuthenticationProvider())
.saml2Login(saml2 -> {
try {
saml2
.loginPage("/saml2")
// Add this
.relyingPartyRegistrationRepository(relyingPartyRegistrations())
.authenticationRequestResolver(new OpenSaml4AuthenticationRequestResolver(
relyingPartyRegistrations()
))
.successHandler(
new CustomSaml2AuthenticationSuccessHandler(
loginAttemptService,
@ -258,9 +282,52 @@ public class SecurityConfiguration {
userService))
.failureHandler(
new CustomSaml2AuthenticationFailureHandler())
.permitAll())
.addFilterBefore(
userAuthenticationFilter, Saml2WebSsoAuthenticationFilter.class);
.permitAll();
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}).addFilterBefore(samlDebugFilter, SecurityContextHolderFilter.class)
.saml2Logout(logout -> logout
.logoutUrl("/logout"))
;
http.addFilterBefore(new OncePerRequestFilter() {
@Override
protected void doFilterInternal(HttpServletRequest request,
HttpServletResponse response, FilterChain chain)
throws ServletException, IOException {
if (request.getRequestURI().contains("/saml2/authenticate")) {
log.info("SAML Auth Request - URI: " + request.getRequestURI());
log.info("SAML Auth Request - Method: " + request.getMethod());
log.info("SAML Auth Request - Query String: " + request.getQueryString());
// Log all request parameters
request.getParameterMap().forEach((key, value) -> {
log.info("SAML Auth Request - Parameter - " + key + ": " + Arrays.toString(value));
});
// Log request content if POST
if ("POST".equalsIgnoreCase(request.getMethod())) {
try {
BufferedReader reader = request.getReader();
StringBuilder sb = new StringBuilder();
String line;
while ((line = reader.readLine()) != null) {
sb.append(line);
}
log.info("SAML Auth Request - Body: " + sb.toString());
} catch (Exception e) {
log.info("Could not read request body", e);
}
}
}
chain.doFilter(request, response);
}
}, Saml2WebSsoAuthenticationRequestFilter.class);
}
} else {
if (applicationProperties.getSecurity().getCsrfDisabled()) {
@ -282,19 +349,27 @@ public class SecurityConfiguration {
return http.build();
}
@Bean
@ConditionalOnProperty(
name = "security.saml2.enabled",
havingValue = "true",
matchIfMissing = false)
public AuthenticationProvider samlAuthenticationProvider() {
OpenSaml4AuthenticationProvider authenticationProvider =
new OpenSaml4AuthenticationProvider();
authenticationProvider.setResponseAuthenticationConverter(
new CustomSaml2ResponseAuthenticationConverter(userService));
return authenticationProvider;
}
@Bean
public AuthenticationProvider samlAuthenticationProvider() {
OpenSaml4AuthenticationProvider provider = new OpenSaml4AuthenticationProvider();
provider.setResponseAuthenticationConverter(
new CustomSaml2ResponseAuthenticationConverter(userService));
provider.setAssertionValidator(token -> {
try {
HashMap<String, Object> params = new HashMap<>();
// Add 5 minutes clock skew
params.put(Saml2ErrorCodes.INVALID_ASSERTION, Duration.ofMinutes(5));
ValidationContext context = new ValidationContext(params);
return Saml2ResponseValidatorResult.success();
} catch (Exception e) {
return Saml2ResponseValidatorResult.failure();
}
});
return provider;
}
// Client Registration Repository for OAUTH2 OIDC Login
@Bean
@ConditionalOnProperty(
@ -432,39 +507,35 @@ public class SecurityConfiguration {
havingValue = "true",
matchIfMissing = false)
public RelyingPartyRegistrationRepository relyingPartyRegistrations() throws Exception {
SAML2 samlConf = applicationProperties.getSecurity().getSaml2();
Resource privateKeyResource = samlConf.getPrivateKey();
X509Certificate idpCert = CertificateUtils.readCertificate(samlConf.getidpCert());
Saml2X509Credential verificationCredential = Saml2X509Credential.verification(idpCert);
Resource privateKeyResource = samlConf.getPrivateKey();
Resource certificateResource = samlConf.getSpCert();
Saml2X509Credential signingCredential =
new Saml2X509Credential(
Saml2X509Credential signingCredential = new Saml2X509Credential(
CertificateUtils.readPrivateKey(privateKeyResource),
CertificateUtils.readCertificate(certificateResource),
Saml2X509CredentialType.SIGNING);
X509Certificate idpCert = CertificateUtils.readCertificate(samlConf.getidpCert());
Saml2X509Credential verificationCredential = Saml2X509Credential.verification(idpCert);
RelyingPartyRegistration rp =
RelyingPartyRegistration.withRegistrationId(samlConf.getRegistrationId())
.signingX509Credentials((c) -> c.add(signingCredential))
.assertingPartyMetadata(
(details) ->
details.entityId(samlConf.getIdpIssuer())
.singleSignOnServiceLocation(
samlConf.getIdpSingleLoginUrl())
.verificationX509Credentials(
(c) -> c.add(verificationCredential))
.wantAuthnRequestsSigned(true))
RelyingPartyRegistration rp = RelyingPartyRegistration.withRegistrationId(samlConf.getRegistrationId())
.assertionConsumerServiceLocation("{baseUrl}/login/saml2/sso/stirlingpdf-saml")
.entityId("http://localhost:8080/saml2/service-provider-metadata/stirlingpdf-saml")
.signingX509Credentials(c -> c.add(signingCredential))
.assertingPartyDetails(party -> party
.entityId(samlConf.getIdpIssuer())
.singleSignOnServiceLocation(samlConf.getIdpSingleLoginUrl())
.verificationX509Credentials(c -> c.add(verificationCredential))
.singleSignOnServiceBinding(Saml2MessageBinding.POST) // Add this
.wantAuthnRequestsSigned(true)
)
.build();
return new InMemoryRelyingPartyRegistrationRepository(rp);
}
@Bean
public DaoAuthenticationProvider daoAuthenticationProvider() {
DaoAuthenticationProvider provider = new DaoAuthenticationProvider();
provider.setUserDetailsService(userDetailsService);

88
src/main/java/stirling/software/SPDF/config/security/saml2/CustomSaml2ResponseAuthenticationConverter.java
View File

@ -30,57 +30,91 @@ public class CustomSaml2ResponseAuthenticationConverter
this.userService = userService;
}
private Map<String, List<Object>> extractAttributes(Assertion assertion) {
Map<String, List<Object>> attributes = new HashMap<>();
for (AttributeStatement attributeStatement : assertion.getAttributeStatements()) {
for (Attribute attribute : attributeStatement.getAttributes()) {
String attributeName = attribute.getName();
List<Object> values = new ArrayList<>();
for (XMLObject xmlObject : attribute.getAttributeValues()) {
// Get the text content directly
String value = xmlObject.getDOM().getTextContent();
if (value != null && !value.trim().isEmpty()) {
values.add(value);
}
}
if (!values.isEmpty()) {
// Store with both full URI and last part of the URI
attributes.put(attributeName, values);
String shortName = attributeName.substring(attributeName.lastIndexOf('/') + 1);
attributes.put(shortName, values);
}
}
}
return attributes;
}
@Override
public Saml2Authentication convert(ResponseToken responseToken) {
// Extract the assertion from the response
Assertion assertion = responseToken.getResponse().getAssertions().get(0);
Map<String, List<Object>> attributes = extractAttributes(assertion);
// Extract the NameID
String nameId = assertion.getSubject().getNameID().getValue();
// Debug log with actual values
log.debug("Extracted SAML Attributes: " + attributes);
Optional<User> userOpt = userService.findByUsernameIgnoreCase(nameId);
// Try to get username/identifier in order of preference
String userIdentifier = null;
if (hasAttribute(attributes, "username")) {
userIdentifier = getFirstAttributeValue(attributes, "username");
} else if (hasAttribute(attributes, "emailaddress")) {
userIdentifier = getFirstAttributeValue(attributes, "emailaddress");
} else if (hasAttribute(attributes, "name")) {
userIdentifier = getFirstAttributeValue(attributes, "name");
} else if (hasAttribute(attributes, "upn")) {
userIdentifier = getFirstAttributeValue(attributes, "upn");
} else if (hasAttribute(attributes, "uid")) {
userIdentifier = getFirstAttributeValue(attributes, "uid");
} else {
userIdentifier = assertion.getSubject().getNameID().getValue();
}
// Rest of your existing code...
Optional<User> userOpt = userService.findByUsernameIgnoreCase(userIdentifier);
SimpleGrantedAuthority simpleGrantedAuthority = new SimpleGrantedAuthority("ROLE_USER");
if (userOpt.isPresent()) {
User user = userOpt.get();
if (user != null) {
simpleGrantedAuthority =
new SimpleGrantedAuthority(userService.findRole(user).getAuthority());
simpleGrantedAuthority = new SimpleGrantedAuthority(userService.findRole(user).getAuthority());
}
}
// Extract the SessionIndexes
List<String> sessionIndexes = new ArrayList<>();
for (AuthnStatement authnStatement : assertion.getAuthnStatements()) {
sessionIndexes.add(authnStatement.getSessionIndex());
}
// Extract the Attributes
Map<String, List<Object>> attributes = extractAttributes(assertion);
CustomSaml2AuthenticatedPrincipal principal = new CustomSaml2AuthenticatedPrincipal(
userIdentifier,
attributes,
userIdentifier,
sessionIndexes);
// Create the custom principal
CustomSaml2AuthenticatedPrincipal principal =
new CustomSaml2AuthenticatedPrincipal(nameId, attributes, nameId, sessionIndexes);
// Create the Saml2Authentication
return new Saml2Authentication(
principal,
responseToken.getToken().getSaml2Response(),
Collections.singletonList(simpleGrantedAuthority));
}
private Map<String, List<Object>> extractAttributes(Assertion assertion) {
Map<String, List<Object>> attributes = new HashMap<>();
for (AttributeStatement attributeStatement : assertion.getAttributeStatements()) {
for (Attribute attribute : attributeStatement.getAttributes()) {
String attributeName = attribute.getName();
List<Object> values = new ArrayList<>();
for (XMLObject xmlObject : attribute.getAttributeValues()) {
log.info("BOOL: " + ((XSBoolean) xmlObject).getValue());
values.add(((XSString) xmlObject).getValue());
private boolean hasAttribute(Map<String, List<Object>> attributes, String name) {
return attributes.containsKey(name) && !attributes.get(name).isEmpty();
}
attributes.put(attributeName, values);
}
}
return attributes;
private String getFirstAttributeValue(Map<String, List<Object>> attributes, String name) {
List<Object> values = attributes.get(name);
return values != null && !values.isEmpty() ? values.get(0).toString() : null;
}
}

31
src/main/java/stirling/software/SPDF/config/security/saml2/SamlDebugFilter.java Normal file
View File

@ -0,0 +1,31 @@
package stirling.software.SPDF.config.security.saml2;
import java.io.IOException;
import java.util.Collections;
import org.springframework.stereotype.Component;
import org.springframework.web.filter.OncePerRequestFilter;
import jakarta.servlet.FilterChain;
import jakarta.servlet.ServletException;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import lombok.extern.slf4j.Slf4j;
@Component
@Slf4j
public class SamlDebugFilter extends OncePerRequestFilter {
@Override
protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain filterChain)
throws ServletException, IOException {
if (request.getRequestURI().contains("/saml2/")) {
log.debug("SAML Debug - URI: {}", request.getRequestURI());
log.debug("SAML Debug - Query String: {}", request.getQueryString());
log.debug("SAML Debug - Method: {}", request.getMethod());
Collections.list(request.getHeaderNames()).forEach(headerName ->
log.debug("SAML Debug - Header {}: {}", headerName, request.getHeader(headerName)));
}
filterChain.doFilter(request, response);
}
}

8
src/main/java/stirling/software/SPDF/controller/api/misc/OCRController.java
View File

@ -1,7 +1,5 @@
package stirling.software.SPDF.controller.api.misc;
import io.github.pixee.security.BoundedLineReader;
import io.github.pixee.security.Filenames;
import java.awt.image.BufferedImage;
import java.io.BufferedReader;
import java.io.File;
@ -35,6 +33,8 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import io.github.pixee.security.BoundedLineReader;
import io.github.pixee.security.Filenames;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.extern.slf4j.Slf4j;
@ -176,7 +176,9 @@ public class OCRController {
// Read the final PDF file
byte[] pdfContent = Files.readAllBytes(finalOutputFile);
String outputFilename =
Filenames.toSimpleFileName(inputFile.getOriginalFilename()).replaceFirst("[.][^.]+$", "") + "_OCR.pdf";
Filenames.toSimpleFileName(inputFile.getOriginalFilename())
.replaceFirst("[.][^.]+$", "")
+ "_OCR.pdf";
return ResponseEntity.ok()
.header(

5
src/main/java/stirling/software/SPDF/controller/api/misc/RepairController.java
View File

@ -50,7 +50,6 @@ public class RepairController {
MultipartFile inputFile = request.getFileInput();
// Save the uploaded file to a temporary location
Path tempInputFile = Files.createTempFile("input_", ".pdf");
Path tempOutputFile = Files.createTempFile("output_", ".pdf");
byte[] pdfBytes = null;
inputFile.transferTo(tempInputFile.toFile());
try {
@ -61,14 +60,13 @@ public class RepairController {
command.add("--qdf"); // Linearizes and normalizes PDF structure
command.add("--object-streams=disable"); // Can help with some corruptions
command.add(tempInputFile.toString());
command.add(tempOutputFile.toString());
ProcessExecutorResult returnCode =
ProcessExecutor.getInstance(ProcessExecutor.Processes.QPDF)
.runCommandWithOutputHandling(command);
// Read the optimized PDF file
pdfBytes = pdfDocumentFactory.loadToBytes(tempOutputFile.toFile());
pdfBytes = pdfDocumentFactory.loadToBytes(tempInputFile.toFile());
// Return the optimized PDF as a response
String outputFilename =
@ -79,7 +77,6 @@ public class RepairController {
} finally {
// Clean up the temporary files
Files.deleteIfExists(tempInputFile);
Files.deleteIfExists(tempOutputFile);
}
}
}

3
src/main/resources/application.properties
View File

@ -3,6 +3,9 @@ multipart.enabled=true
logging.level.org.springframework=WARN
logging.level.org.hibernate=WARN
logging.level.org.eclipse.jetty=WARN
logging.level.org.springframework.security.saml2=TRACE
logging.level.org.springframework.security=DEBUG
logging.level.org.opensaml: DEBUG
logging.level.com.zaxxer.hikari=WARN
spring.jpa.open-in-view=false

10
test.sh
View File

@ -73,8 +73,8 @@ main() {
# Building Docker images
# docker build --no-cache --pull --build-arg VERSION_TAG=alpha -t frooodle/s-pdf:latest -f ./Dockerfile .
# docker build --no-cache --pull --build-arg VERSION_TAG=alpha -t frooodle/s-pdf:latest-ultra-lite -f ./Dockerfile-ultra-lite .
# docker build --no-cache --pull --build-arg VERSION_TAG=alpha -t stirlingtools/stirling-pdf:latest -f ./Dockerfile .
# docker build --no-cache --pull --build-arg VERSION_TAG=alpha -t stirlingtools/stirling-pdf:latest-ultra-lite -f ./Dockerfile-ultra-lite .
# Test each configuration
#run_tests "Stirling-PDF-Ultra-Lite" "./exampleYmlFiles/docker-compose-latest-ultra-lite.yml"
@ -93,9 +93,9 @@ main() {
# Building Docker images with security enabled
# docker build --no-cache --pull --build-arg VERSION_TAG=alpha -t frooodle/s-pdf:latest -f ./Dockerfile .
# docker build --no-cache --pull --build-arg VERSION_TAG=alpha -t frooodle/s-pdf:latest-ultra-lite -f ./Dockerfile-ultra-lite .
docker build --no-cache --pull --build-arg VERSION_TAG=alpha -t frooodle/s-pdf:latest-fat -f ./Dockerfile-fat .
# docker build --no-cache --pull --build-arg VERSION_TAG=alpha -t stirlingtools/stirling-pdf:latest -f ./Dockerfile .
# docker build --no-cache --pull --build-arg VERSION_TAG=alpha -t stirlingtools/stirling-pdf:latest-ultra-lite -f ./Dockerfile-ultra-lite .
docker build --no-cache --pull --build-arg VERSION_TAG=alpha -t stirlingtools/stirling-pdf:latest-fat -f ./Dockerfile-fat .
# Test each configuration with security