TEZ-4688 TEZ-4648: Tez upgrade to Hadoop 3.5.0 and jersey 2.x

pom.xml

@@ -81,11 +81,12 @@
     <plexus-velocity.version>2.3.0</plexus-velocity.version>
     <frontend-maven-plugin.version>1.15.0</frontend-maven-plugin.version>
     <guava.version>32.0.1-jre</guava.version>
-    <hadoop.version>3.4.2</hadoop.version>
-    <jersey.version>1.19.4</jersey.version>
+    <hadoop.version>3.5.0</hadoop.version>
+    <jersey.version>2.46</jersey.version>
     <jettison.version>1.5.4</jettison.version>
     <jsr305.version>3.0.0</jsr305.version>
     <junit.version>4.13.2</junit.version>
+    <junit.jupiter.version>5.9.3</junit.jupiter.version>
     <leveldbjni-all.version>1.8</leveldbjni-all.version>
     <lifecycle-mapping.version>1.0.0</lifecycle-mapping.version>
     <metrics-core.version>3.1.0</metrics-core.version>
@@ -328,14 +329,6 @@
             <groupId>commons-logging</groupId>
             <artifactId>commons-logging-api</artifactId>
           </exclusion>
-          <exclusion>
-            <groupId>com.sun.jersey</groupId>
-            <artifactId>jersey-core</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>com.sun.jersey</groupId>
-            <artifactId>jersey-server</artifactId>
-          </exclusion>
           <exclusion>
             <groupId>org.eclipse.jdt</groupId>
             <artifactId>core</artifactId>
@@ -413,14 +406,6 @@
             <groupId>commons-logging</groupId>
             <artifactId>commons-logging-api</artifactId>
           </exclusion>
-          <exclusion>
-            <groupId>com.sun.jersey</groupId>
-            <artifactId>jersey-core</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>com.sun.jersey</groupId>
-            <artifactId>jersey-server</artifactId>
-          </exclusion>
           <exclusion>
             <groupId>io.netty</groupId>
             <artifactId>*</artifactId>
@@ -773,6 +758,18 @@
         <version>${junit.version}</version>
         <scope>test</scope>
       </dependency>
+      <dependency>
+        <groupId>org.junit.jupiter</groupId>
+        <artifactId>junit-jupiter-api</artifactId>
+        <version>${junit.jupiter.version}</version>
+        <scope>test</scope>
+      </dependency>
+      <dependency>
+        <groupId>org.junit.vintage</groupId>
+        <artifactId>junit-vintage-engine</artifactId>
+        <version>${junit.jupiter.version}</version>
+        <scope>test</scope>
+      </dependency>
       <dependency>
         <groupId>com.google.protobuf</groupId>
         <artifactId>protobuf-java</artifactId>
@@ -789,13 +786,13 @@
         <version>${jettison.version}</version>
       </dependency>
       <dependency>
-        <groupId>com.sun.jersey</groupId>
+        <groupId>org.glassfish.jersey.core</groupId>
         <artifactId>jersey-client</artifactId>
         <version>${jersey.version}</version>
       </dependency>
       <dependency>
-        <groupId>com.sun.jersey</groupId>
-        <artifactId>jersey-json</artifactId>
+        <groupId>org.glassfish.jersey.media</groupId>
+        <artifactId>jersey-media-json-jackson</artifactId>
         <version>${jersey.version}</version>
       </dependency>
       <!-- BouncyCastle should be in default scope to make their way to tez.tar.gz since Hadoop 3.4.1 -->

tez-api/pom.xml

@@ -94,12 +94,12 @@
      <artifactId>junit</artifactId>
     </dependency>
     <dependency>
-      <groupId>com.sun.jersey</groupId>
+      <groupId>org.glassfish.jersey.core</groupId>
       <artifactId>jersey-client</artifactId>
     </dependency>
     <dependency>
-      <groupId>com.sun.jersey</groupId>
-      <artifactId>jersey-json</artifactId>
+      <groupId>org.glassfish.jersey.media</groupId>
+      <artifactId>jersey-media-json-jackson</artifactId>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>

tez-api/src/main/java/org/apache/tez/common/ATSConstants.java

@@ -27,15 +27,16 @@ public class ATSConstants {
 
   /* Top level keys */
   public static final String ENTITIES = "entities";
-  public static final String ENTITY = "entity";
-  public static final String ENTITY_TYPE = "entitytype";
+  public static final String ENTITY_ID = "entityId";
+  public static final String ENTITY_TYPE = "entityType";
   public static final String EVENTS = "events";
-  public static final String EVENT_TYPE = "eventtype";
+  public static final String EVENT_TYPE = "eventType";
   public static final String TIMESTAMP = "ts";
-  public static final String EVENT_INFO = "eventinfo";
+  public static final String EVENT_INFO = "eventInfo";
   public static final String RELATED_ENTITIES = "relatedEntities";
-  public static final String PRIMARY_FILTERS = "primaryfilters";
-  public static final String OTHER_INFO = "otherinfo";
+  public static final String PRIMARY_FILTERS = "primaryFilters";
+  public static final String OTHER_INFO = "otherInfo";
+  public static final String DOMAIN_ID = "domainId";
 
   /* Section for related entities */
   public static final String APPLICATION_ID = "applicationId";

tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java

@@ -29,6 +29,8 @@
 import java.util.Set;
 
 import javax.annotation.Nullable;
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.WebTarget;
 import javax.ws.rs.core.MediaType;
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -52,11 +54,6 @@
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Joiner;
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.ClientHandlerException;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.UniformInterfaceException;
-import com.sun.jersey.api.client.WebResource;
 
 import org.codehaus.jettison.json.JSONArray;
 import org.codehaus.jettison.json.JSONException;
@@ -212,7 +209,7 @@ public DAGStatus waitForCompletionWithStatusUpdates(
  @Override
   public void close() throws IOException {
     if (httpClient != null) {
-      httpClient.destroy();
+      httpClient.close();
       httpClient = null;
     }
     if (timelineReaderStrategy != null) {
@@ -426,26 +423,14 @@ protected Map<String, VertexTaskStats> parseTaskStatsForVertexes()
   @VisibleForTesting
   protected JSONObject getJsonRootEntity(String url) throws TezException {
     try {
-      WebResource wr = getCachedHttpClient().resource(url);
-      ClientResponse response = wr.accept(MediaType.APPLICATION_JSON_TYPE)
-          .type(MediaType.APPLICATION_JSON_TYPE)
-          .get(ClientResponse.class);
-
-      final ClientResponse.Status clientResponseStatus = response.getClientResponseStatus();
-      if (clientResponseStatus != ClientResponse.Status.OK) {
-        throw new TezException("Failed to get response from YARN Timeline:" +
-            " errorCode:" + clientResponseStatus + ", url:" + url);
-      }
-
-      return response.getEntity(JSONObject.class);
-    } catch (ClientHandlerException e) {
+      Client client = getCachedHttpClient();
+      WebTarget target = client.target(url);
+      String json = target.request(MediaType.APPLICATION_JSON_TYPE)
+          .accept(MediaType.APPLICATION_JSON_TYPE)
+          .get(String.class);
+      return new JSONObject(json);
+    } catch (Exception e) {
       throw new TezException("Error processing response from YARN Timeline", e);
-    } catch (UniformInterfaceException e) {
-      throw new TezException("Error accessing content from YARN Timeline - unexpected response", e);
-    } catch (IllegalArgumentException e) {
-      throw new TezException("Error accessing content from YARN Timeline - invalid url", e);
-    } catch (IOException e) {
-      throw new TezException("Error failed to get http client", e);
     }
   }
 

tez-api/src/main/java/org/apache/tez/dag/api/client/TimelineReaderFactory.java

@@ -30,25 +30,23 @@
 import javax.net.ssl.HostnameVerifier;
 import javax.net.ssl.HttpsURLConnection;
 import javax.net.ssl.SSLSocketFactory;
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.ClientBuilder;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
 import org.apache.hadoop.security.authentication.client.Authenticator;
 import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
 import org.apache.hadoop.security.ssl.SSLFactory;
 import org.apache.tez.common.ReflectionUtils;
 import org.apache.tez.dag.api.TezException;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.config.ClientConfig;
-import com.sun.jersey.api.client.config.DefaultClientConfig;
-import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
-import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
-import com.sun.jersey.json.impl.provider.entity.JSONRootElementProvider;
 
+import org.glassfish.jersey.client.ClientConfig;
+import org.glassfish.jersey.client.HttpUrlConnectorProvider;
+import org.glassfish.jersey.jackson.JacksonFeature;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -164,7 +162,7 @@ public Client getHttpClient() throws IOException {
       UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
       UserGroupInformation realUgi = ugi.getRealUser();
       String doAsUser;
-      ClientConfig clientConfig = new DefaultClientConfig(JSONRootElementProvider.App.class);
+      ClientConfig clientConfig = new ClientConfig().register(JacksonFeature.class);
       ConnectionConfigurator connectionConfigurator = getNewConnectionConf(useHttps,
               connTimeout, sslFactory);
 
@@ -181,14 +179,16 @@ public Client getHttpClient() throws IOException {
         doAsUser = null;
       }
 
-      HttpURLConnectionFactory connectionFactory;
+      HttpUrlConnectorProvider.ConnectionFactory connectionFactory;
       try {
         connectionFactory = new TokenAuthenticatedURLConnectionFactory(connectionConfigurator, authenticator,
                 doAsUser);
       } catch (TezException e) {
         throw new IOException("Fail to create TokenAuthenticatedURLConnectionFactory", e);
       }
-      return new Client(new URLConnectionClientHandler(connectionFactory), clientConfig);
+
+      clientConfig.connectorProvider(new HttpUrlConnectorProvider().connectionFactory(connectionFactory));
+      return ClientBuilder.newClient(clientConfig);
     }
 
     private static Authenticator getTokenAuthenticator() throws TezException {
@@ -203,12 +203,12 @@ private static Authenticator getTokenAuthenticator() throws TezException {
       return ReflectionUtils.createClazzInstance(authenticatorClazzName);
     }
 
-    private static class TokenAuthenticatedURLConnectionFactory implements HttpURLConnectionFactory {
+    private static class TokenAuthenticatedURLConnectionFactory implements HttpUrlConnectorProvider.ConnectionFactory {
 
       private final Authenticator authenticator;
       private final ConnectionConfigurator connConfigurator;
       private final String doAsUser;
-      private final AuthenticatedURL.Token token;
+      private final Object token;
 
       public TokenAuthenticatedURLConnectionFactory(ConnectionConfigurator connConfigurator,
                                                     Authenticator authenticator,
@@ -221,9 +221,9 @@ public TokenAuthenticatedURLConnectionFactory(ConnectionConfigurator connConfigu
       }
 
       @Override
-      public HttpURLConnection getHttpURLConnection(URL url) throws IOException {
+      public HttpURLConnection getConnection(URL url) throws IOException {
         try {
-          AuthenticatedURL authenticatedURL= ReflectionUtils.createClazzInstance(
+          HttpURLConnection authenticatedURL = ReflectionUtils.createClazzInstance(
               DELEGATION_TOKEN_AUTHENTICATED_URL_CLAZZ_NAME, new Class[] {
               delegationTokenAuthenticatorClazz,
               ConnectionConfigurator.class
@@ -265,21 +265,24 @@ public TimelineReaderPseudoAuthenticatedStrategy(final Configuration conf,
 
     @Override
     public Client getHttpClient() {
-      ClientConfig config = new DefaultClientConfig(JSONRootElementProvider.App.class);
-      HttpURLConnectionFactory urlFactory = new PseudoAuthenticatedURLConnectionFactory(connectionConf);
-      return new Client(new URLConnectionClientHandler(urlFactory), config);
+      ClientConfig config = new ClientConfig().register(JacksonFeature.class);
+      HttpUrlConnectorProvider.ConnectionFactory urlFactory =
+          new PseudoAuthenticatedURLConnectionFactory(connectionConf);
+      config.connectorProvider(new HttpUrlConnectorProvider().connectionFactory(urlFactory));
+      return ClientBuilder.newClient(config);
     }
 
     @VisibleForTesting
-    protected static class PseudoAuthenticatedURLConnectionFactory implements HttpURLConnectionFactory {
+    protected static class PseudoAuthenticatedURLConnectionFactory
+        implements HttpUrlConnectorProvider.ConnectionFactory {
       private final ConnectionConfigurator connectionConf;
 
       public PseudoAuthenticatedURLConnectionFactory(ConnectionConfigurator connectionConf) {
         this.connectionConf = connectionConf;
       }
 
       @Override
-      public HttpURLConnection getHttpURLConnection(URL url) throws IOException {
+      public HttpURLConnection getConnection(URL url) throws IOException {
         String tokenString = (url.getQuery() == null ? "?" : "&") + "user.name=" +
             URLEncoder.encode(UserGroupInformation.getCurrentUser().getShortUserName(), "UTF8");
 

tez-api/src/test/java/org/apache/tez/dag/api/client/TestATSHttpClient.java

@@ -95,7 +95,7 @@ public void testGetDagStatusSimple() throws TezException, JSONException, IOExcep
 
     final String jsonDagData =
             "{ " +
-            "  otherinfo: { " +
+            "  otherInfo: { " +
             "    status: 'SUCCEEDED'," +
             "    diagnostics: 'SAMPLE_DIAGNOSTICS'," +
             "    counters: { counterGroups: [ " +
@@ -108,9 +108,9 @@ public void testGetDagStatusSimple() throws TezException, JSONException, IOExcep
             "}";
 
     final String jsonVertexData = "{entities:[ " +
-        "{otherinfo: {vertexName:'v1', numTasks:5,numFailedTasks:1,numSucceededTasks:2," +
+        "{otherInfo: {vertexName:'v1', numTasks:5,numFailedTasks:1,numSucceededTasks:2," +
           "numKilledTasks:3,numCompletedTasks:3}}," +
-        "{otherinfo: {vertexName:'v2',numTasks:10,numFailedTasks:1,numSucceededTasks:5," +
+        "{otherInfo: {vertexName:'v2',numTasks:10,numFailedTasks:1,numSucceededTasks:5," +
           "numKilledTasks:3,numCompletedTasks:4}}" +
         "]}";
 
@@ -151,7 +151,7 @@ public void testGetVertexStatusSimple() throws JSONException, TezException, IOEx
     Set<StatusGetOpts> statusOptions = new HashSet<StatusGetOpts>(1);
     statusOptions.add(StatusGetOpts.GET_COUNTERS);
 
-    final String jsonData = "{entities:[ {otherinfo:{numFailedTasks:1,numSucceededTasks:2," +
+    final String jsonData = "{entities:[ {otherInfo:{numFailedTasks:1,numSucceededTasks:2," +
         "status:'SUCCEEDED', vertexName:'vertex1name', numTasks:4, numKilledTasks: 3, " +
         "numCompletedTasks: 4, diagnostics: 'diagnostics1', " +
         "counters: { counterGroups: [ " +

tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java

@@ -21,7 +21,7 @@
 import static org.mockito.Mockito.mock;
 
 import java.net.HttpURLConnection;
-import java.net.URL;
+import java.net.URI;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -52,10 +52,10 @@ public void testPseudoAuthenticatorConnectionUrlShouldHaveUserName() throws Exce
     ConnectionConfigurator connConf = mock(ConnectionConfigurator.class);
     TimelineReaderPseudoAuthenticatedStrategy.PseudoAuthenticatedURLConnectionFactory
         connectionFactory = new TimelineReaderPseudoAuthenticatedStrategy
-          .PseudoAuthenticatedURLConnectionFactory(connConf);
+        .PseudoAuthenticatedURLConnectionFactory(connConf);
     String inputUrl = "http://host:8080/path";
     String expectedUrl = inputUrl + "?user.name=" + UserGroupInformation.getCurrentUser().getShortUserName();
-    HttpURLConnection httpURLConnection = connectionFactory.getHttpURLConnection(new URL(inputUrl));
+    HttpURLConnection httpURLConnection = connectionFactory.getConnection(URI.create(inputUrl).toURL());
     Assert.assertEquals(expectedUrl, httpURLConnection.getURL().toString());
   }
 

tez-dag/pom.xml

@@ -161,6 +161,11 @@
       <artifactId>curator-test</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>