Skip to content
This repository was archived by the owner on Oct 8, 2020. It is now read-only.

Commit 9ea180a

Browse files
patrickwestphalpatrickwestphal
authored
Merge pull request #6 from SANSA-Stack/feature/dependency-updates
Feature/dependency updates
2 parents 3114ad8 + 660f832 commit 9ea180a

11 files changed

Lines changed: 258 additions & 103 deletions

File tree

‎pom.xml‎

Lines changed: 49 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -66,22 +66,22 @@
6666
<!--<java.version>>=1.8</java.version>-->
6767
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
6868
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
69-
<scala.version>2.11.11</scala.version>
69+
<scala.version>2.11.12</scala.version>
7070
<scala.binary.version>2.11</scala.binary.version>
71-
<spark.version>2.2.1</spark.version>
72-
<flink.version>1.4.0</flink.version>
73-
<jena.version>3.5.0</jena.version>
71+
<spark.version>2.3.1</spark.version>
72+
<flink.version>1.5.0</flink.version>
73+
<jena.version>3.6.0</jena.version>
7474
<sansa.stack.version>0.3.1-SNAPSHOT</sansa.stack.version>
7575
<sansa.rdf.version>${sansa.stack.version}</sansa.rdf.version>
7676
<sansa.query.version>${sansa.stack.version}</sansa.query.version>
7777
<sansa.owl.version>${sansa.stack.version}</sansa.owl.version>
78-
<jsa.subversion>1</jsa.subversion>
78+
<jsa.subversion>3</jsa.subversion>
7979
<jsa.version>${jena.version}-${jsa.subversion}</jsa.version>
8080
<PermGen>64m</PermGen>
8181
<MaxPermGen>512m</MaxPermGen>
8282
<CodeCacheSize>512m</CodeCacheSize>
8383
<gpg.keyname>AKSW</gpg.keyname>
84-
<owlapi.version>5.1.3</owlapi.version>
84+
<owlapi.version>5.1.5</owlapi.version>
8585
<scalastyle.config.path>${project.basedir}/scalastyle-config.xml</scalastyle.config.path>
8686
</properties>
8787

@@ -96,6 +96,15 @@
9696
<groupId>${project.groupId}</groupId>
9797
<artifactId>sansa-rdf-spark_${scala.binary.version}</artifactId>
9898
<version>${sansa.rdf.version}</version>
99+
<!-- we have to exclude an old version here transitively included by a Kafka dependency of Spark
100+
Don't know how Spark 2.3 is able to work with Kafka 0.10 right now ...
101+
-->
102+
<exclusions>
103+
<exclusion>
104+
<groupId>net.jpountz.lz4</groupId>
105+
<artifactId>lz4</artifactId>
106+
</exclusion>
107+
</exclusions>
99108
</dependency>
100109
<dependency>
101110
<groupId>${project.groupId}</groupId>
@@ -149,6 +158,16 @@
149158
<artifactId>spark-sql_${scala.binary.version}</artifactId>
150159
<version>${spark.version}</version>
151160
</dependency>
161+
<dependency>
162+
<groupId>org.apache.spark</groupId>
163+
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
164+
<version>${spark.version}</version>
165+
</dependency>
166+
<dependency>
167+
<groupId>org.apache.spark</groupId>
168+
<artifactId>spark-streaming-kafka-0-10_${scala.binary.version}</artifactId>
169+
<version>${spark.version}</version>
170+
</dependency>
152171

153172
<!-- Apache Flink -->
154173
<dependency>
@@ -178,6 +197,16 @@
178197
<artifactId>jena-arq</artifactId>
179198
<version>${jena.version}</version>
180199
</dependency>
200+
<dependency>
201+
<groupId>org.apache.jena</groupId>
202+
<artifactId>jena-tdb</artifactId>
203+
<version>${jena.version}</version>
204+
</dependency>
205+
<dependency>
206+
<groupId>org.apache.jena</groupId>
207+
<artifactId>jena-cmds</artifactId>
208+
<version>${jena.version}</version>
209+
</dependency>
181210

182211
<!-- OWL API -->
183212
<dependency>
@@ -203,24 +232,29 @@
203232

204233
<!-- Graph API -->
205234
<dependency>
206-
<groupId>com.assembla.scala-incubator</groupId>
235+
<groupId>org.scala-graph</groupId>
207236
<artifactId>graph-core_${scala.binary.version}</artifactId>
208-
<version>1.10.0</version>
237+
<version>1.12.3</version>
209238
</dependency>
210239
<dependency>
211-
<groupId>com.assembla.scala-incubator</groupId>
240+
<groupId>org.scala-graph</groupId>
212241
<artifactId>graph-dot_${scala.binary.version}</artifactId>
213-
<version>1.9.0</version>
242+
<version>1.11.5</version>
214243
</dependency>
215244
<dependency>
216245
<groupId>org.jgrapht</groupId>
217246
<artifactId>jgrapht-core</artifactId>
218-
<version>1.1.0</version>
247+
<version>1.2.0</version>
248+
</dependency>
249+
<dependency>
250+
<groupId>org.jgrapht</groupId>
251+
<artifactId>jgrapht-io</artifactId>
252+
<version>1.2.0</version>
219253
</dependency>
220254
<dependency>
221255
<groupId>org.jgrapht</groupId>
222256
<artifactId>jgrapht-ext</artifactId>
223-
<version>1.1.0</version>
257+
<version>1.2.0</version>
224258
</dependency>
225259
<dependency>
226260
<groupId>org.gephi</groupId>
@@ -265,7 +299,7 @@
265299
<dependency>
266300
<groupId>com.typesafe.scala-logging</groupId>
267301
<artifactId>scala-logging_${scala.binary.version}</artifactId>
268-
<version>3.7.2</version>
302+
<version>3.9.0</version>
269303
</dependency>
270304

271305
<!-- Guava -->
@@ -279,7 +313,7 @@
279313
<dependency>
280314
<groupId>com.chuusai</groupId>
281315
<artifactId>shapeless_${scala.binary.version}</artifactId>
282-
<version>2.3.2</version>
316+
<version>2.3.3</version>
283317
</dependency>
284318

285319
<!-- Scopt -->
@@ -293,7 +327,7 @@
293327
<dependency>
294328
<groupId>com.typesafe</groupId>
295329
<artifactId>config</artifactId>
296-
<version>1.3.2</version>
330+
<version>1.3.3</version>
297331
</dependency>
298332

299333

‎sansa-inference-common/pom.xml‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,21 +31,19 @@
3131
<dependency>
3232
<groupId>org.apache.jena</groupId>
3333
<artifactId>jena-tdb</artifactId>
34-
<version>3.5.0</version>
3534
</dependency>
3635
<dependency>
3736
<groupId>org.apache.jena</groupId>
3837
<artifactId>jena-cmds</artifactId>
39-
<version>3.5.0</version>
4038
</dependency>
4139

4240
<!-- Graph API -->
4341
<dependency>
44-
<groupId>com.assembla.scala-incubator</groupId>
42+
<groupId>org.scala-graph</groupId>
4543
<artifactId>graph-core_${scala.binary.version}</artifactId>
4644
</dependency>
4745
<dependency>
48-
<groupId>com.assembla.scala-incubator</groupId>
46+
<groupId>org.scala-graph</groupId>
4947
<artifactId>graph-dot_${scala.binary.version}</artifactId>
5048
</dependency>
5149
<dependency>
@@ -56,6 +54,10 @@
5654
<groupId>org.jgrapht</groupId>
5755
<artifactId>jgrapht-ext</artifactId>
5856
</dependency>
57+
<dependency>
58+
<groupId>org.jgrapht</groupId>
59+
<artifactId>jgrapht-io</artifactId>
60+
</dependency>
5961
<dependency>
6062
<groupId>org.gephi</groupId>
6163
<artifactId>gephi-toolkit</artifactId>
@@ -65,7 +67,7 @@
6567
<artifactId>google-collections</artifactId>
6668
</exclusion>
6769
</exclusions>
68-
<scope>provided</scope>
70+
<scope>compile</scope>
6971
</dependency>
7072

7173
<!-- Guava -->
@@ -85,6 +87,11 @@
8587
<version>3.5.0</version>
8688
</dependency>
8789

90+
<!-- Scopt CLI API-->
91+
<dependency>
92+
<groupId>com.github.scopt</groupId>
93+
<artifactId>scopt_${scala.binary.version}</artifactId>
94+
</dependency>
8895

8996
<!-- Test -->
9097
<dependency>

‎sansa-inference-common/src/main/resources/log4j.properties‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# Root logger option
2-
log4j.rootLogger=INFO, stdout
2+
log4j.rootLogger=DEBUG, stdout
33

44
# Direct log messages to a log file
55
log4j.appender.file=org.apache.log4j.RollingFileAppender
@@ -20,3 +20,5 @@ log4j.logger.akka.remote.Remoting=ERROR
2020

2121
log4j.logger.org.apache.hadoop=ERROR
2222
log4j.logger.org.apache.calcite=ERROR
23+
24+
log4j.logger.scalax.collection.connectivity.GraphComponents=OFF

‎sansa-inference-common/src/main/scala/net/sansa_stack/inference/rules/RuleDependencyGraphAnalyzer.scala‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -137,8 +137,8 @@ object RuleDependencyGraphAnalyzer extends Logging{
137137
def main(args: Array[String]) {
138138
// we re-use the JENA API for parsing rules
139139
val filenames = List(
140-
// "rules/rdfs-simple.rules"
141-
"rules/owl_horst.rules"
140+
"rules/rdfs-simple.rules"
141+
// "rules/owl_horst.rules"
142142
// "rules/owl_rl.rules"
143143
)
144144

@@ -154,7 +154,7 @@ object RuleDependencyGraphAnalyzer extends Logging{
154154

155155
// print each rule as graph
156156
rules.foreach { r =>
157-
val g = RuleUtils.asGraph(r).export(new File(graphDir, r.getName + ".graphml").toString)
157+
RuleUtils.asGraph(r).export(new File(graphDir, r.getName + ".graphml").toString)
158158
}
159159

160160
// generate graph

‎sansa-inference-common/src/main/scala/net/sansa_stack/inference/rules/RuleDependencyGraphGenerator.scala‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import java.util.stream.Collectors
55
import scala.collection
66
import scala.collection.JavaConverters._
77
import scala.language.{existentials, implicitConversions}
8+
89
import scalax.collection.GraphPredef._
910
import scalax.collection.GraphTraversal.Parameters
1011
import scalax.collection._
@@ -13,7 +14,6 @@ import scalax.collection.edge._
1314
import scalax.collection.mutable.DefaultGraphImpl
1415
import scalax.collection.GraphPredef._
1516
import scalax.collection.GraphEdge._
16-
1717
import org.apache.jena.graph.{Node, NodeFactory}
1818
import org.apache.jena.reasoner.TriplePattern
1919
import org.apache.jena.reasoner.rulesys.Rule
@@ -259,12 +259,12 @@ object RuleDependencyGraphGenerator extends Logging {
259259
pairsOfRules :+= (cycle.last, cycle(0))
260260

261261
// map to list of edges
262-
val edges: Buffer[graph.EdgeT] = pairsOfRules.map(e => {
262+
val edges: Buffer[graph.EdgeT] = pairsOfRules.flatMap(e => {
263263
val node1 = graph get e._1
264264
val node2 = graph get e._2
265265

266266
node1.outgoing.filter(_.target == node2)
267-
}).flatten
267+
})
268268
debug("Edges: " + edges.mkString(", "))
269269

270270
// map to edge labels, i.e. the predicates

‎sansa-inference-common/src/main/scala/net/sansa_stack/inference/rules/minimizer/RuleDependencyGraphMinimizer.scala‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
package net.sansa_stack.inference.rules.minimizer
22
import scala.collection.mutable.{ArrayBuffer, Buffer}
3+
34
import scalax.collection.Graph
45
import scalax.collection.edge.LDiEdge
56
import scala.collection.JavaConverters._
6-
import scalax.collection.GraphTraversal.Parameters
7+
import scala.collection.mutable
78

9+
import scalax.collection.GraphTraversal.Parameters
810
import org.apache.jena.graph.{Node, NodeFactory}
911
import org.apache.jena.reasoner.TriplePattern
1012
import org.apache.jena.reasoner.rulesys.Rule
@@ -16,7 +18,6 @@ import net.sansa_stack.inference.rules.RuleDependencyGraphGenerator.{asString, d
1618
import net.sansa_stack.inference.utils.{GraphUtils, RuleUtils}
1719
import net.sansa_stack.inference.utils.graph.LabeledEdge
1820
import net.sansa_stack.inference.utils.RuleUtils._
19-
2021
import scalax.collection.GraphTraversal.Parameters
2122
import scalax.collection._
2223
import scalax.collection.edge.Implicits._
@@ -294,11 +295,13 @@ abstract class RuleDependencyGraphMinimizer extends MinimizationRuleExecutor {
294295
// debug(cycles.asScala.mkString(","))
295296

296297
// cycles that contain the current node
297-
val cyclesWithNode: Buffer[Buffer[Rule]] = allCycles.asScala.filter(cycle => cycle.contains(node.value)).map(cycle => cycle.asScala)
298+
val cyclesWithNode: mutable.Buffer[mutable.Buffer[Rule]] = allCycles.asScala
299+
.filter(cycle => cycle.contains(node.value))
300+
.map(cycle => cycle.asScala)
298301
debug("Cycles: " + cyclesWithNode.map(c => c.map(r => r.getName)).mkString(","))
299302

300303
// cycles that use the same property
301-
val cyclesWithNodeSameProp: Map[Node, scala.List[Buffer[graph.EdgeT]]] = cyclesWithNode.map(cycle => {
304+
val cyclesWithNodeSameProp: Map[Node, scala.List[mutable.Buffer[graph.EdgeT]]] = cyclesWithNode.map(cycle => {
302305

303306
debug("Cycle: " + cycle.map(r => r.getName).mkString(", "))
304307

@@ -307,12 +310,12 @@ abstract class RuleDependencyGraphMinimizer extends MinimizationRuleExecutor {
307310
pairsOfRules :+= (cycle.last, cycle(0))
308311

309312
// map to list of edges
310-
val edges: Buffer[graph.EdgeT] = pairsOfRules.map(e => {
313+
val edges: mutable.Buffer[graph.EdgeT] = pairsOfRules.flatMap(e => {
311314
val node1 = graph get e._1
312315
val node2 = graph get e._2
313316

314317
node1.outgoing.filter(_.target == node2)
315-
}).flatten
318+
})
316319
debug("Edges: " + edges.mkString(", "))
317320

318321
// map to edge labels, i.e. the predicates
@@ -325,9 +328,14 @@ abstract class RuleDependencyGraphMinimizer extends MinimizationRuleExecutor {
325328
if (samePred) Some(predicates(0), edges) else None
326329
}).filter(_.isDefined).map(_.get).groupBy(e => e._1).mapValues(e => e.map(x => x._2).toList)
327330

328-
var removedCycles: collection.mutable.Set[Buffer[graph.EdgeT]] = collection.mutable.Set()
331+
var removedCycles: collection.mutable.Set[mutable.Buffer[graph.EdgeT]] = collection.mutable.Set()
329332

330-
val tmp: Map[Node, Map[Int, List[Buffer[graph.EdgeT]]]] = cyclesWithNodeSameProp.mapValues(value => value.map(cycle => (cycle.size, cycle)).groupBy(_._1).mapValues(e => e.map(x => x._2).toList))
333+
val tmp: Map[Node, Map[Int, List[mutable.Buffer[graph.EdgeT]]]] =
334+
cyclesWithNodeSameProp
335+
.mapValues(value =>
336+
value.map(cycle => (cycle.size, cycle))
337+
.groupBy(_._1)
338+
.mapValues(e => e.map(x => x._2)))
331339

332340
tmp.foreach(predicate2Cycles => {
333341
debug("predicate: " + predicate2Cycles._1)

0 commit comments

Comments
 (0)