Merge pull request #118 from IABTechLab/ian-UID2-6154-optout-e2e-update-sqs

update e2e tests for optout sqs

Author: Ian-Nara

Dockerfile

@@ -22,8 +22,10 @@ ENV E2E_PHONE_SUPPORT ""
 
 ENV UID2_CORE_E2E_OPERATOR_API_KEY ""
 ENV UID2_CORE_E2E_OPTOUT_API_KEY ""
+ENV UID2_CORE_E2E_OPTOUT_INTERNAL_API_KEY ""
 ENV UID2_CORE_E2E_CORE_URL ""
 ENV UID2_CORE_E2E_OPTOUT_URL ""
+ENV UID2_CORE_E2E_LOCALSTACK_URL ""
 
 ENV UID2_OPERATOR_E2E_CLIENT_SITE_ID ""
 ENV UID2_OPERATOR_E2E_CLIENT_API_KEY ""

docker-compose.yml

@@ -3,24 +3,27 @@ version: "3.8"
 services:
   localstack:
     container_name: localstack
-    image: localstack/localstack:1.3.0
+    image: localstack/localstack:4.0.3
     ports:
       - "127.0.0.1:5001:5001"
     volumes:
       - "./docker/uid2-core/src/init-aws.sh:/etc/localstack/init/ready.d/init-aws-core.sh"
       - "./docker/uid2-core/src/s3/core:/s3/core"
-      - "./docker/uid2-core/src/kms/seed.yaml:/init/seed.yaml"
       - "./docker/uid2-optout/src/init-aws.sh:/etc/localstack/init/ready.d/init-aws-optout.sh"
       - "./docker/uid2-optout/src/s3/optout:/s3/optout"
     environment:
-      - EDGE_PORT=5001
-      - KMS_PROVIDER=local-kms
+      - GATEWAY_LISTEN=0.0.0.0:5001
+      - LOCALSTACK_HOST=localstack:5001
+      - SERVICES=s3,sqs,kms
+      - DEFAULT_REGION=us-east-1
     healthcheck:
       test: awslocal s3api wait bucket-exists --bucket test-core-bucket
         && awslocal s3api wait bucket-exists --bucket test-optout-bucket
+        && awslocal sqs get-queue-url --queue-name optout-queue
+        && awslocal kms describe-key --key-id ff275b92-0def-4dfc-b0f6-87c96b26c6c7
       interval: 5s
-      timeout: 5s
-      retries: 3
+      timeout: 10s
+      retries: 6
     networks:
       - e2e_default
 
@@ -49,17 +52,20 @@ services:
     image: ghcr.io/iabtechlab/uid2-optout:latest
     ports:
       - "127.0.0.1:8081:8081"
+      - "127.0.0.1:8082:8082"
       - "127.0.0.1:5090:5005"
     volumes:
       - ./docker/uid2-optout/conf/default-config.json:/app/conf/default-config.json
       - ./docker/uid2-optout/conf/local-e2e-docker-config.json:/app/conf/local-config.json
       - ./docker/uid2-optout/mount/:/opt/uid2/optout/
     depends_on:
+      localstack:
+        condition: service_healthy
       core:
         condition: service_healthy
     healthcheck:
       test: wget --tries=1 --spider http://localhost:8081/ops/healthcheck || exit 1
-      interval: 5s
+      retries: 12
     networks:
       - e2e_default
 

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.uid2</groupId>
     <artifactId>uid2-e2e</artifactId>
-    <version>4.1.0</version>
+    <version>4.1.15-alpha-89-SNAPSHOT</version>
 
     <properties>
         <maven.compiler.source>21</maven.compiler.source>

src/test/java/app/component/Optout.java

@@ -0,0 +1,108 @@
+package app.component;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.uid2.shared.util.Mapper;
+import common.Const;
+import common.EnvUtil;
+import common.HttpClient;
+
+/**
+ * Component for interacting with the UID2 Optout service.
+ */
+public class Optout extends App {
+    private static final ObjectMapper OBJECT_MAPPER = Mapper.getInstance();
+    
+    // The SQS delta producer runs on port 8082 (8081 + 1)
+    private static final int DELTA_PRODUCER_PORT_OFFSET = 1;
+    
+    private String optoutInternalApiKey;
+
+    public Optout(String host, Integer port, String name) {
+        super(host, port, name);
+        this.optoutInternalApiKey = EnvUtil.getEnv(Const.Config.Core.OPTOUT_INTERNAL_API_KEY, false);
+    }
+
+    public Optout(String host, String name) {
+        super(host, null, name);
+        this.optoutInternalApiKey = EnvUtil.getEnv(Const.Config.Core.OPTOUT_INTERNAL_API_KEY, false);
+    }
+    
+    private String getOptoutInternalApiKey() {
+        if (optoutInternalApiKey == null || optoutInternalApiKey.isEmpty()) {
+            throw new IllegalStateException("Missing environment variable: " + Const.Config.Core.OPTOUT_INTERNAL_API_KEY);
+        }
+        return optoutInternalApiKey;
+    }
+
+    /**
+     * Triggers delta production on the optout service.
+     * This reads from the SQS queue and produces delta files.
+     * The endpoint is on port 8082 (optout port + 1).
+     * 
+     * @return JsonNode with response, or null if job already running (409)
+     */
+    public JsonNode triggerDeltaProduce() throws Exception {
+        String deltaProduceUrl = getDeltaProducerBaseUrl() + "/optout/deltaproduce";
+        try {
+            String response = HttpClient.post(deltaProduceUrl, "", getOptoutInternalApiKey());
+            return OBJECT_MAPPER.readTree(response);
+        } catch (HttpClient.HttpException e) {
+            if (e.getCode() == 409) {
+                // Job already running - this is fine, we'll just wait for it
+                return null;
+            }
+            throw e;
+        }
+    }
+
+    /**
+     * Gets the status of the current delta production job.
+     */
+    public JsonNode getDeltaProduceStatus() throws Exception {
+        String statusUrl = getDeltaProducerBaseUrl() + "/optout/deltaproduce/status";
+        String response = HttpClient.get(statusUrl, getOptoutInternalApiKey());
+        return OBJECT_MAPPER.readTree(response);
+    }
+
+    /**
+     * Triggers delta production and waits for it to complete.
+     * If a job is already running, waits for that job instead.
+     * @param maxWaitSeconds Maximum time to wait for completion
+     * @return true if delta production completed successfully
+     */
+    public boolean triggerDeltaProduceAndWait(int maxWaitSeconds) throws Exception {
+        // Try to trigger - will return null if job already running (409)
+        triggerDeltaProduce();
+        
+        long startTime = System.currentTimeMillis();
+        long maxWaitMs = maxWaitSeconds * 1000L;
+        
+        while (System.currentTimeMillis() - startTime < maxWaitMs) {
+            Thread.sleep(2000); // Poll every 2 seconds
+            
+            JsonNode status = getDeltaProduceStatus();
+            String state = status.path("state").asText();
+            
+            if ("completed".equalsIgnoreCase(state) || "failed".equalsIgnoreCase(state)) {
+                return "completed".equalsIgnoreCase(state);
+            }
+            
+            // If idle (no job), try to trigger again
+            if ("idle".equalsIgnoreCase(state) || "none".equalsIgnoreCase(state) || state.isEmpty()) {
+                triggerDeltaProduce();
+            }
+        }
+        
+        return false; // Timed out
+    }
+
+    private String getDeltaProducerBaseUrl() {
+        // Delta producer runs on optout port + 1
+        if (getPort() != null) {
+            return "http://" + getHost() + ":" + (getPort() + DELTA_PRODUCER_PORT_OFFSET);
+        }
+        // If port not specified, assume default optout port (8081) + 1
+        return "http://" + getHost() + ":8082";
+    }
+}

src/test/java/common/Const.java

@@ -13,6 +13,7 @@ public static final class Config {
         public static final class Core {
             public static final String OPERATOR_API_KEY = "UID2_CORE_E2E_OPERATOR_API_KEY";
             public static final String OPTOUT_API_KEY = "UID2_CORE_E2E_OPTOUT_API_KEY";
+            public static final String OPTOUT_INTERNAL_API_KEY = "UID2_CORE_E2E_OPTOUT_INTERNAL_API_KEY";
             public static final String CORE_URL = "UID2_CORE_E2E_CORE_URL";
             public static final String OPTOUT_URL = "UID2_CORE_E2E_OPTOUT_URL";
         }

src/test/java/common/KmsHelper.java

@@ -0,0 +1,82 @@
+package common;
+
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.kms.KmsClient;
+import software.amazon.awssdk.services.kms.model.GetPublicKeyRequest;
+import software.amazon.awssdk.services.kms.model.GetPublicKeyResponse;
+
+import java.net.URI;
+import java.security.KeyFactory;
+import java.security.PublicKey;
+import java.security.spec.X509EncodedKeySpec;
+import java.util.Base64;
+
+/**
+ * Helper class for interacting with KMS (or LocalStack KMS) in e2e tests.
+ * 
+ * This allows tests to dynamically fetch public keys from KMS rather than
+ * relying on hardcoded keys, which is necessary when using LocalStack since
+ * it generates its own RSA key material.
+ */
+public final class KmsHelper {
+    
+    private static final String LOCALSTACK_ENDPOINT = "http://localstack:5001";
+    private static final String KMS_KEY_ID = "ff275b92-0def-4dfc-b0f6-87c96b26c6c7";
+    private static final Region REGION = Region.US_EAST_1;
+    
+    private KmsHelper() {
+    }
+    
+    /**
+     * Fetches the public key from LocalStack KMS for the configured key ID.
+     * 
+     * @return The public key as a base64-encoded string
+     * @throws Exception if the key cannot be fetched or parsed
+     */
+    public static String getPublicKeyFromLocalstack() throws Exception {
+        try (KmsClient kmsClient = createLocalstackKmsClient()) {
+            GetPublicKeyRequest request = GetPublicKeyRequest.builder()
+                    .keyId(KMS_KEY_ID)
+                    .build();
+            
+            GetPublicKeyResponse response = kmsClient.getPublicKey(request);
+            byte[] publicKeyBytes = response.publicKey().asByteArray();
+            
+            // Return as base64-encoded string (format expected by JwtService)
+            return Base64.getEncoder().encodeToString(publicKeyBytes);
+        }
+    }
+    
+    /**
+     * Fetches the public key from LocalStack KMS and returns it as a Java PublicKey object.
+     * 
+     * @return The PublicKey object
+     * @throws Exception if the key cannot be fetched or parsed
+     */
+    public static PublicKey getPublicKeyObjectFromLocalstack() throws Exception {
+        try (KmsClient kmsClient = createLocalstackKmsClient()) {
+            GetPublicKeyRequest request = GetPublicKeyRequest.builder()
+                    .keyId(KMS_KEY_ID)
+                    .build();
+            
+            GetPublicKeyResponse response = kmsClient.getPublicKey(request);
+            
+            byte[] publicKeyBytes = response.publicKey().asByteArray();
+            
+            KeyFactory keyFactory = KeyFactory.getInstance("RSA");
+            X509EncodedKeySpec keySpec = new X509EncodedKeySpec(publicKeyBytes);
+            return keyFactory.generatePublic(keySpec);
+        }
+    }
+    
+    private static KmsClient createLocalstackKmsClient() {
+        return KmsClient.builder()
+                .endpointOverride(URI.create(LOCALSTACK_ENDPOINT))
+                .region(REGION)
+                .credentialsProvider(StaticCredentialsProvider.create(
+                        AwsBasicCredentials.create("test", "test")))
+                .build();
+    }
+}

src/test/java/suite/core/CoreTest.java

@@ -1,10 +1,11 @@
 package suite.core;
 
 import common.HttpClient;
+import common.KmsHelper;
 import app.component.Core;
 import com.fasterxml.jackson.databind.JsonNode;
+import com.uid2.shared.Const;
 import com.uid2.shared.attest.JwtService;
-import com.uid2.shared.attest.JwtValidationResponse;
 import io.vertx.core.json.JsonObject;
 import org.junit.jupiter.api.condition.EnabledIf;
 import org.junit.jupiter.params.ParameterizedTest;
@@ -29,6 +30,13 @@ public void testAttest_EmptyAttestationRequest(Core core) {
         assertEquals("Unsuccessful POST request - URL: " + coreUrl + "/attest - Code: 400 Bad Request - Response body: {\"status\":\"no attestation_request attached\"}", exception.getMessage());
     }
 
+    /**
+     * Tests valid attestation request with JWT signing.
+     * 
+     * Since LocalStack generates its own RSA key material,
+     * we dynamically fetch the public key from LocalStack's
+     * KMS using GetPublicKey API to validate JWT signatures.
+     */
     @ParameterizedTest(name = "/attest - {0}")
     @MethodSource({
             "suite.core.TestData#baseArgs"
@@ -38,7 +46,7 @@ public void testAttest_ValidAttestationRequest(Core core) throws Exception {
 
         JsonNode response = core.attest(validTrustedAttestationRequest);
 
-        assertAll("",
+        assertAll("Attestation response should be successful",
                 () -> assertNotNull(response.get("status")),
                 () -> assertEquals("success", response.get("status").asText()));
 
@@ -48,29 +56,45 @@ public void testAttest_ValidAttestationRequest(Core core) throws Exception {
                 () -> assertNotNull(body.get("attestation_token")),
                 () -> assertNotNull(body.get("expiresAt")));
 
-        JwtService jwtService = new JwtService(getConfig());
-        assertNotNull(body.get("attestation_jwt_optout"));
-        JwtValidationResponse validationResponseOptOut = jwtService.validateJwt(body.get("attestation_jwt_optout").asText(), Core.OPTOUT_URL, Core.CORE_URL);
-        assertAll("testAttest_ValidAttestationRequest valid OptOut JWT. Local OptOut URL: '" + Core.OPTOUT_URL + "', Core URL: '" + Core.CORE_URL + "'",
-                () -> assertNotNull(validationResponseOptOut),
-                () -> assertTrue(validationResponseOptOut.getIsValid()));
+        // Verify JWTs are generated - LocalStack 4.x supports KMS Sign
+        JsonNode jwtOptoutNode = body.get("attestation_jwt_optout");
+        JsonNode jwtCoreNode = body.get("attestation_jwt_core");
+        
+        assertAll("JWTs should be generated by KMS Sign",
+                () -> assertNotNull(jwtOptoutNode, "attestation_jwt_optout should not be null"),
+                () -> assertFalse(jwtOptoutNode.isNull(), "attestation_jwt_optout should not be JSON null"),
+                () -> assertFalse(jwtOptoutNode.asText().isEmpty(), "attestation_jwt_optout should not be empty"),
+                () -> assertNotNull(jwtCoreNode, "attestation_jwt_core should not be null"),
+                () -> assertFalse(jwtCoreNode.isNull(), "attestation_jwt_core should not be JSON null"),
+                () -> assertFalse(jwtCoreNode.asText().isEmpty(), "attestation_jwt_core should not be empty"));
 
-        assertNotNull(body.get("attestation_jwt_core"));
-        JwtValidationResponse validationResponseCore = jwtService.validateJwt(body.get("attestation_jwt_core").asText(), Core.CORE_URL, Core.CORE_URL);
-        assertAll("testAttest_ValidAttestationRequest valid Core JWT. Local Core URL: '" + Core.CORE_URL + "'",
-                () -> assertNotNull(validationResponseCore),
-                () -> assertTrue(validationResponseCore.getIsValid()));
+        // Verify JWT format (header.payload.signature)
+        String jwtOptout = jwtOptoutNode.asText();
+        String jwtCore = jwtCoreNode.asText();
+        assertAll("JWTs should have valid format",
+                () -> assertEquals(3, jwtOptout.split("\\.").length, "OptOut JWT should have 3 parts"),
+                () -> assertEquals(3, jwtCore.split("\\.").length, "Core JWT should have 3 parts"));
+
+        // Fetch the public key dynamically from LocalStack KMS and validate JWT signatures
+        String publicKeyBase64 = KmsHelper.getPublicKeyFromLocalstack();
+        JsonObject config = new JsonObject()
+                .put(Const.Config.AwsKmsJwtSigningPublicKeysProp, publicKeyBase64);
+        JwtService jwtService = new JwtService(config);
+        
+        // Validate optout JWT signature
+        var optoutValidation = jwtService.validateJwt(jwtOptout, Core.OPTOUT_URL, Core.CORE_URL);
+        assertTrue(optoutValidation.getIsValid(), "OptOut JWT signature should be valid");
+        
+        // Validate core JWT signature  
+        var coreValidation = jwtService.validateJwt(jwtCore, Core.CORE_URL, Core.CORE_URL);
+        assertTrue(coreValidation.getIsValid(), "Core JWT signature should be valid");
 
         String optoutUrl = body.get("optout_url").asText();
         assertAll("testAttest_ValidAttestationRequest OptOut URL not null",
                 () -> assertNotNull(optoutUrl),
                 () -> assertEquals(Core.OPTOUT_URL, optoutUrl));
     }
 
-    private static JsonObject getConfig() {
-        return new JsonObject("{  \"aws_kms_jwt_signing_public_keys\": \"MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAmvwB41qI5Fe41PDbXqcX5uOvSvfKh8l9QV0O3M+NsB4lKqQEP0t1hfoiXTpOgKz1ArYxHsQ2LeXifX4uwEbYJFlpVM+tyQkTWQjBOw6fsLYK2Xk4X2ylNXUUf7x3SDiOVxyvTh3OZW9kqrDBN9JxSoraNLyfw0hhW0SHpfs699SehgbQ7QWep/gVlKRLIz0XAXaZNw24s79ORcQlrCE6YD0PgQmpI/dK5xMML82n6y3qcTlywlGaU7OGIMdD+CTXA3BcOkgXeqZTXNaX1u6jCTa1lvAczun6avp5VZ4TFiuPo+y4rJ3GU+14cyT5NckEcaTKSvd86UdwK5Id9tl3bQIDAQAB\"}");
-    }
-
     @ParameterizedTest(name = "/operator/config - {0}")
     @MethodSource({
             "suite.core.TestData#baseArgs"