Skip to content
This repository was archived by the owner on Oct 8, 2020. It is now read-only.

Commit d4dda38

Browse files
LorenzBuehmannLorenzBuehmann
committed
Flink/Spark CLI tarball generation.
1 parent 6b06c38 commit d4dda38

5 files changed

Lines changed: 253 additions & 19 deletions

File tree

‎sansa-inference-flink/pom.xml‎

Lines changed: 169 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -390,24 +390,8 @@ under the License.
390390
<profile>
391391
<id>dist-package</id>
392392
<build>
393+
<finalName>SANSA-Inference-Flink-${project.version}</finalName>
393394
<plugins>
394-
<plugin>
395-
<artifactId>maven-assembly-plugin</artifactId>
396-
<configuration>
397-
<descriptors>
398-
<descriptor>src/assembly/assembly-bin.xml</descriptor>
399-
</descriptors>
400-
</configuration>
401-
<executions>
402-
<execution>
403-
<id>make-assembly</id> <!-- this is used for inheritance merges -->
404-
<phase>package</phase> <!-- bind to the packaging phase -->
405-
<goals>
406-
<goal>single</goal>
407-
</goals>
408-
</execution>
409-
</executions>
410-
</plugin>
411395
<plugin>
412396
<artifactId>maven-resources-plugin</artifactId>
413397
<executions>
@@ -448,6 +432,174 @@ under the License.
448432
</execution>
449433
</executions>
450434
</plugin>
435+
<plugin>
436+
<groupId>org.apache.maven.plugins</groupId>
437+
<artifactId>maven-shade-plugin</artifactId>
438+
<executions>
439+
<!-- Run shade goal on package phase -->
440+
<execution>
441+
<phase>package</phase>
442+
<goals>
443+
<goal>shade</goal>
444+
</goals>
445+
<configuration>
446+
<!--<minimizeJar>true</minimizeJar>-->
447+
<artifactSet>
448+
<excludes>
449+
<!-- This list contains all dependencies of flink-dist
450+
Everything else will be packaged into the fat-jar
451+
-->
452+
<exclude>org.apache.flink:flink-shaded-*_2.11</exclude>
453+
<exclude>org.apache.flink:flink-core_2.11</exclude>
454+
<exclude>org.apache.flink:flink-java*</exclude>
455+
<exclude>org.apache.flink:flink-java_2.11</exclude>
456+
<exclude>org.apache.flink:flink-scala_2.11</exclude>
457+
<exclude>org.apache.flink:flink-runtime_2.11</exclude>
458+
<exclude>org.apache.flink:flink-optimizer_2.11</exclude>
459+
<exclude>org.apache.flink:flink-clients_2.11</exclude>
460+
<exclude>org.apache.flink:flink-avro_2.11</exclude>
461+
<exclude>org.apache.flink:flink-java-examples_2.11</exclude>
462+
<exclude>org.apache.flink:flink-scala-examples_2.11</exclude>
463+
<exclude>org.apache.flink:flink-streaming-examples_2.11</exclude>
464+
<exclude>org.apache.flink:flink-streaming-java_2.11</exclude>
465+
<exclude>org.apache.flink:*</exclude>
466+
467+
<!-- Also exclude very big transitive dependencies of Flink
468+
469+
WARNING: You have to remove these excludes if your code relies on other
470+
versions of these dependencies.
471+
472+
-->
473+
474+
<exclude>org.scala-lang:scala-library</exclude>
475+
<exclude>org.scala-lang:scala-compiler</exclude>
476+
<exclude>org.scala-lang:scala-reflect</exclude>
477+
<exclude>com.amazonaws:aws-java-sdk</exclude>
478+
<exclude>com.typesafe.akka:akka-actor_*</exclude>
479+
<exclude>com.typesafe.akka:akka-remote_*</exclude>
480+
<exclude>com.typesafe.akka:akka-slf4j_*</exclude>
481+
<exclude>io.netty:netty-all</exclude>
482+
<exclude>io.netty:netty</exclude>
483+
<exclude>org.eclipse.jetty:jetty-server</exclude>
484+
<exclude>org.eclipse.jetty:jetty-continuation</exclude>
485+
<exclude>org.eclipse.jetty:jetty-http</exclude>
486+
<exclude>org.eclipse.jetty:jetty-io</exclude>
487+
<exclude>org.eclipse.jetty:jetty-util</exclude>
488+
<exclude>org.eclipse.jetty:jetty-security</exclude>
489+
<exclude>org.eclipse.jetty:jetty-servlet</exclude>
490+
<exclude>org.gephi:*</exclude>
491+
<exclude>org.netbeans.api:*</exclude>
492+
493+
<exclude>commons-fileupload:commons-fileupload</exclude>
494+
<exclude>org.apache.avro:avro</exclude>
495+
<exclude>commons-collections:commons-collections</exclude>
496+
<exclude>org.codehaus.jackson:jackson-core-asl</exclude>
497+
<exclude>org.codehaus.jackson:jackson-mapper-asl</exclude>
498+
<exclude>com.thoughtworks.paranamer:paranamer</exclude>
499+
<exclude>org.xerial.snappy:snappy-java</exclude>
500+
<exclude>org.apache.commons:commons-compress</exclude>
501+
<exclude>org.tukaani:xz</exclude>
502+
<exclude>com.esotericsoftware.kryo:kryo</exclude>
503+
<exclude>com.esotericsoftware.minlog:minlog</exclude>
504+
<exclude>org.objenesis:objenesis</exclude>
505+
<exclude>com.twitter:chill_*</exclude>
506+
<exclude>com.twitter:chill-java</exclude>
507+
<exclude>com.twitter:chill-avro_*</exclude>
508+
<exclude>com.twitter:chill-bijection_*</exclude>
509+
<exclude>com.twitter:bijection-core_*</exclude>
510+
<exclude>com.twitter:bijection-avro_*</exclude>
511+
<exclude>commons-lang:commons-lang</exclude>
512+
<exclude>junit:junit</exclude>
513+
<exclude>de.javakaffee:kryo-serializers</exclude>
514+
<exclude>joda-time:joda-time</exclude>
515+
<exclude>org.apache.commons:commons-lang3</exclude>
516+
<exclude>org.slf4j:slf4j-api</exclude>
517+
<exclude>org.slf4j:slf4j-log4j12</exclude>
518+
<exclude>log4j:log4j</exclude>
519+
<exclude>org.apache.commons:commons-math</exclude>
520+
<exclude>org.apache.sling:org.apache.sling.commons.json</exclude>
521+
<exclude>commons-logging:commons-logging</exclude>
522+
<exclude>org.apache.httpcomponents:httpclient</exclude>
523+
<exclude>org.apache.httpcomponents:httpcore</exclude>
524+
<exclude>commons-codec:commons-codec</exclude>
525+
<exclude>com.fasterxml.jackson.core:jackson-core</exclude>
526+
<exclude>com.fasterxml.jackson.core:jackson-databind</exclude>
527+
<exclude>com.fasterxml.jackson.core:jackson-annotations</exclude>
528+
<exclude>org.codehaus.jettison:jettison</exclude>
529+
<exclude>stax:stax-api</exclude>
530+
<exclude>com.typesafe:config</exclude>
531+
<exclude>org.uncommons.maths:uncommons-maths</exclude>
532+
<!--<exclude>com.github.scopt:scopt_*</exclude>-->
533+
<exclude>org.mortbay.jetty:servlet-api</exclude>
534+
<exclude>commons-io:commons-io</exclude>
535+
<exclude>commons-cli:commons-cli</exclude>
536+
537+
<exclude>sqlite:*</exclude>
538+
<exclude>netbeans:*</exclude>
539+
<exclude>org.apache.xmlgraphics:*</exclude>
540+
<exclude>mysql:*</exclude>
541+
<exclude>jfree:*</exclude>
542+
<exclude>org.scala-lang.modules:*</exclude>
543+
</excludes>
544+
</artifactSet>
545+
<filters>
546+
<filter>
547+
<artifact>org.apache.flink:*</artifact>
548+
<excludes>
549+
<exclude>org/apache/flink/shaded/**</exclude>
550+
<exclude>web-docs/**</exclude>
551+
</excludes>
552+
</filter>
553+
<filter>
554+
<!-- Do not copy the signatures in the META-INF folder.
555+
Otherwise, this might cause SecurityExceptions when using the JAR. -->
556+
<artifact>*:*</artifact>
557+
<excludes>
558+
<exclude>META-INF/*.SF</exclude>
559+
<exclude>META-INF/*.DSA</exclude>
560+
<exclude>META-INF/*.RSA</exclude>
561+
</excludes>
562+
</filter>
563+
</filters>
564+
<finalName>dist-${project.artifactId}-${project.version}</finalName>
565+
<outputDirectory>${project.build.directory}/shaded</outputDirectory>
566+
<transformers>
567+
<transformer
568+
implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
569+
<!-- add Main-Class to manifest file -->
570+
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
571+
<mainClass>net.sansa_stack.inference.flink.RDFGraphMaterializer</mainClass>
572+
</transformer>
573+
</transformers>
574+
<createDependencyReducedPom>false</createDependencyReducedPom>
575+
<relocations>
576+
<relocation>
577+
<pattern>scopt</pattern>
578+
<shadedPattern>scopt.shaded</shadedPattern>
579+
</relocation>
580+
</relocations>
581+
</configuration>
582+
</execution>
583+
</executions>
584+
</plugin>
585+
<plugin>
586+
<artifactId>maven-assembly-plugin</artifactId>
587+
<configuration>
588+
<descriptors>
589+
<descriptor>src/assembly/assembly-bin.xml</descriptor>
590+
</descriptors>
591+
<appendAssemblyId>false</appendAssemblyId>
592+
</configuration>
593+
<executions>
594+
<execution>
595+
<id>make-assembly</id> <!-- this is used for inheritance merges -->
596+
<phase>package</phase> <!-- bind to the packaging phase -->
597+
<goals>
598+
<goal>single</goal>
599+
</goals>
600+
</execution>
601+
</executions>
602+
</plugin>
451603
</plugins>
452604
</build>
453605
</profile>

‎sansa-inference-flink/scripts/cli‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,14 @@
1717
# limitations under the License.
1818
#
1919

20+
# get location of the script
21+
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
22+
2023
# version will be overwritten my Maven-Antrun-Plugin
2124
INFERENCE_LIB_JAR=VERSION_TMP
2225

2326
# the path to the Inference distribution Jar file relative to the current
24-
INFERENCE_LIB=./lib/$INFERENCE_LIB_JAR
27+
INFERENCE_LIB=$DIR/../lib/$INFERENCE_LIB_JAR
2528

2629
# check whether environment variable FLINK_HOME was set
2730
if [ -z "${FLINK_HOME}" ]; then

‎sansa-inference-flink/src/assembly/assembly-bin.xml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.0.0"
2+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
3+
xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.0.0 http://maven.apache.org/xsd/assembly-2.0.0.xsd">
4+
<id>bin</id>
5+
<formats>
6+
<format>tar.gz</format>
7+
<!--<format>tar.bz2</format>-->
8+
<!--<format>zip</format>-->
9+
</formats>
10+
<fileSets>
11+
<fileSet>
12+
<directory>${project.basedir}</directory>
13+
<outputDirectory></outputDirectory>
14+
<includes>
15+
<include>README*</include>
16+
<include>LICENSE*</include>
17+
<include>NOTICE*</include>
18+
</includes>
19+
</fileSet>
20+
<fileSet>
21+
<directory>${project.build.directory}/shaded</directory>
22+
<outputDirectory>lib</outputDirectory>
23+
<includes>
24+
<include>*.jar</include>
25+
</includes>
26+
</fileSet>
27+
<fileSet>
28+
<directory>${project.build.directory}/scripts</directory>
29+
<outputDirectory>bin</outputDirectory>
30+
<includes>
31+
<include>cli</include>
32+
</includes>
33+
<fileMode>0755</fileMode>
34+
</fileSet>
35+
<fileSet>
36+
<directory>${project.build.directory}/site</directory>
37+
<outputDirectory>docs</outputDirectory>
38+
</fileSet>
39+
</fileSets>
40+
</assembly>

‎sansa-inference-spark/pom.xml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -398,8 +398,8 @@
398398
<profile>
399399
<id>dist-package</id>
400400
<build>
401+
<finalName>SANSA-Inference-Spark-${project.version}</finalName>
401402
<plugins>
402-
403403
<plugin>
404404
<artifactId>maven-resources-plugin</artifactId>
405405
<executions>
@@ -548,6 +548,8 @@
548548
<finalName>dist-${project.artifactId}-${project.version}</finalName>
549549
<outputDirectory>${project.build.directory}/shaded</outputDirectory>
550550
<transformers>
551+
<transformer
552+
implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
551553
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
552554
<mainClass>net.sansa_stack.inference.spark.RDFGraphMaterializer</mainClass>
553555
</transformer>
@@ -564,6 +566,7 @@
564566
<descriptors>
565567
<descriptor>src/assembly/assembly-bin.xml</descriptor>
566568
</descriptors>
569+
<appendAssemblyId>false</appendAssemblyId>
567570
</configuration>
568571
<executions>
569572
<execution>

‎sansa-inference-spark/scripts/cli‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/usr/bin/env bash
2+
3+
#
4+
# Licensed to the Apache Software Foundation (ASF) under one or more
5+
# contributor license agreements. See the NOTICE file distributed with
6+
# this work for additional information regarding copyright ownership.
7+
# The ASF licenses this file to You under the Apache License, Version 2.0
8+
# (the "License"); you may not use this file except in compliance with
9+
# the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
#
19+
20+
# get location of the script
21+
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
22+
23+
# version will be overwritten my Maven-Antrun-Plugin
24+
INFERENCE_LIB_JAR=VERSION_TMP
25+
26+
# the path to the Inference distribution Jar file relative to the current
27+
INFERENCE_LIB=$DIR/../lib/$INFERENCE_LIB_JAR
28+
29+
# check whether environment variable SPARK_HOME was set
30+
if [ -z "${SPARK_HOME}" ]; then
31+
echo "You have to set the environment variable SPARK_HOME, which maps to the home directory of the Spark distribution."
32+
exit 1
33+
fi
34+
35+
# submit the Spark job
36+
exec "${SPARK_HOME}"/bin/spark-submit "$INFERENCE_LIB" "$@"

0 commit comments

Comments
 (0)