Skip to content

Commit fe65758

Browse files
derlinderlin
committed
FIX BUG: stats should also be saved if no write + fix tests + accept text/csv
- n_reads stats were not saved properly in MySQL - cassandra stats returned wrong values when no write - make last_ts NULL in MySQL so n_reads can be saved without writes - fix bug in test (+= vs =+) - change test rest template to add accept and Content-Type headers to json by default - make values endpoint work with Content-Types text/plain, test/csv and application/csv
1 parent 8ef1079 commit fe65758

11 files changed

Lines changed: 100 additions & 62 deletions

File tree

‎.gitignore‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,9 @@ out/
3131
### VS Code ###
3232
.vscode/
3333

34-
### Mac ###
34+
### Mac ###
3535
.DS_STORE
3636

3737
### Other ###
38-
other/cassandra/data
38+
other/cassandra/data
39+
logs

‎other/mysql/bbdata2-structure.sql‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
-- MySQL Script generated by MySQL Workbench
2-
-- Wed Sep 23 16:33:54 2020
2+
-- Wed Sep 23 19:11:54 2020
33
-- Model: New Model Version: 1.0
44
-- MySQL Workbench Forward Engineering
55

@@ -320,7 +320,7 @@ CREATE TABLE IF NOT EXISTS `stats` (
320320
`object_id` INT(11) NOT NULL,
321321
`n_reads` INT NOT NULL DEFAULT 0,
322322
`n_writes` INT NOT NULL DEFAULT 0,
323-
`last_ts` TIMESTAMP(3) NOT NULL,
323+
`last_ts` TIMESTAMP(3) NULL,
324324
`avg_sample_period` DOUBLE NOT NULL DEFAULT 0,
325325
`created` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
326326
`last_modified` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,

‎other/mysql/bbdata2.mwb‎

-1 Bytes
Binary file not shown.

‎src/main/kotlin/ch/derlin/bbdata/common/cassandra/Streamers.kt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@ class CassandraObjectStreamer(private val mapper: ObjectMapper) {
2222
response: HttpServletResponse,
2323
csvHeaders: List<String>,
2424
values: Iterable<StreamableCsv>) {
25-
if (contentType != null && contentType.contains("text")) {
25+
if (contentType != null &&
26+
(contentType.contains("text") || contentType.contains("csv"))) {
2627
response.contentType = "text/csv"
2728
streamCsv(response.outputStream, csvHeaders, values)
2829
} else {

‎src/main/kotlin/ch/derlin/bbdata/common/stats/SqlStatsRepository.kt‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,4 @@ import org.springframework.transaction.annotation.Transactional
1515

1616
@Profile(Profiles.SQL_STATS)
1717
@Repository
18-
interface SqlStatsRepository : JpaRepository<SqlStats, Long> {
19-
20-
@Transactional
21-
@Modifying
22-
@Query("UPDATE SqlStats s set s.nReads = s.nReads + 1 WHERE s.objectId = :objectId")
23-
fun incrementReadCounter(objectId: Long)
24-
}
18+
interface SqlStatsRepository : JpaRepository<SqlStats, Long>

‎src/main/kotlin/ch/derlin/bbdata/common/stats/StatsLogic.kt‎

Lines changed: 22 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -62,24 +62,25 @@ class CassandraStatsLogic(private val objectStatsRepository: ObjectStatsReposito
6262
objectStatsCounterRepository.updateWriteCounter(v.objectId.toInt())
6363
}
6464

65-
override fun incrementReadCounter(objectId: Long) =
66-
objectStatsCounterRepository.updateReadCounter(objectId.toInt())
65+
override fun incrementReadCounter(objectId: Long) {
6766

68-
override fun getStats(objectId: Long): Stats {
69-
var stats: Stats? = null
70-
objectStatsRepository.findById(objectId.toInt()).ifPresent { os ->
71-
objectStatsCounterRepository.findById(objectId.toInt()).ifPresent { oc ->
72-
stats = Stats(
73-
objectId = objectId,
74-
nReads = oc.nReads,
75-
nWrites = oc.nValues,
76-
lastTs = os.lastTimestamp,
77-
avgSamplePeriod = os.avgSamplePeriod.toDouble()
78-
)
79-
}
80-
}
81-
return stats ?: Stats(objectId = objectId)
67+
objectStatsCounterRepository.updateReadCounter(objectId.toInt())
8268
}
69+
70+
override fun getStats(objectId: Long): Stats =
71+
objectStatsRepository.findById(objectId.toInt()).let { osp ->
72+
val os = if (osp.isPresent) osp.get() else null
73+
objectStatsCounterRepository.findById(objectId.toInt()).let { ocp ->
74+
val oc = if (ocp.isPresent) ocp.get() else null
75+
Stats(
76+
objectId = objectId,
77+
nReads = oc?.nReads ?: 0,
78+
nWrites = oc?.nValues ?: 0,
79+
lastTs = os?.lastTimestamp,
80+
avgSamplePeriod = os?.avgSamplePeriod?.toDouble() ?: .0
81+
)
82+
}
83+
}
8384
}
8485

8586
@Component
@@ -101,8 +102,11 @@ class SqlStatsLogic(private val statsRepository: SqlStatsRepository) : StatsLogi
101102
})
102103
}
103104

104-
override fun incrementReadCounter(objectId: Long) =
105-
statsRepository.incrementReadCounter(objectId)
105+
override fun incrementReadCounter(objectId: Long) {
106+
val stats = statsRepository.findById(objectId).orElseGet{ SqlStats(objectId = objectId) }
107+
stats.nReads += 1
108+
statsRepository.save(stats)
109+
}
106110

107111
override fun getStats(objectId: Long): Stats {
108112
var stats: Stats? = null

‎src/main/kotlin/ch/derlin/bbdata/output/api/values/ValuesController.kt‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,10 @@ class ValuesController(
3939

4040

4141
@Protected
42-
@GetMapping("/objects/{objectId}/values", produces = ["application/json", "text/plain"])
42+
@GetMapping("/objects/{objectId}/values", produces = ["application/json", "text/csv", "text/plain", "application/csv"])
4343
@ApiResponse(content = [
4444
Content(mediaType = "application/json", array = ArraySchema(schema = Schema(implementation = RawValue::class))),
45-
Content(mediaType = "text/plain", schema = Schema(example = RawValue.csvHeadersString))
45+
Content(mediaType = "text/csv", schema = Schema(example = RawValue.csvHeadersString))
4646
])
4747
fun getRawValuesStream(
4848
@UserId userId: Int,
@@ -64,10 +64,10 @@ class ValuesController(
6464
}
6565

6666
@Protected
67-
@GetMapping("/objects/{objectId}/values/latest", produces = ["application/json", "text/plain"])
67+
@GetMapping("/objects/{objectId}/values/latest", produces = ["application/json", "text/csv", "text/plain", "application/csv"])
6868
@ApiResponse(content = [
6969
Content(mediaType = "application/json", array = ArraySchema(schema = Schema(implementation = RawValue::class))),
70-
Content(mediaType = "text/plain", schema = Schema(example = RawValue.csvHeadersString))])
70+
Content(mediaType = "text/csv", schema = Schema(example = RawValue.csvHeadersString))])
7171
fun getLatestValue(
7272
@UserId userId: Int,
7373
@CType contentType: String,
@@ -88,10 +88,10 @@ class ValuesController(
8888
}
8989

9090
@Protected
91-
@GetMapping("/objects/{objectId}/values/aggregated", produces = ["application/json", "text/plain"])
91+
@GetMapping("/objects/{objectId}/values/aggregated", produces = ["application/json", "text/csv", "text/plain", "application/csv"])
9292
@ApiResponse(content = [
9393
Content(mediaType = "application/json", array = ArraySchema(schema = Schema(implementation = Aggregation::class))),
94-
Content(mediaType = "text/plain", schema = Schema(example = Aggregation.csvHeadersString))
94+
Content(mediaType = "text/csv", schema = Schema(example = Aggregation.csvHeadersString))
9595
])
9696
fun getQuarterAggregationsStream(
9797
@UserId userId: Int,

‎src/test/kotlin/ch/derlin/bbdata/TestRestTemplateExtensions.kt‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -14,23 +14,23 @@ import org.springframework.web.client.RestClientException
1414

1515
object JsonEntity {
1616

17-
fun jsonHeaders(): HttpHeaders {
17+
fun jsonHeaders(vararg additionalHeaders: Pair<String, Any?>): HttpHeaders {
1818
val headers = HttpHeaders()
19-
headers.add("Content-Type", MediaType.APPLICATION_JSON_VALUE)
19+
if (additionalHeaders.find { it.first.equals("accept", ignoreCase = true) } == null)
20+
headers.add("accept", MediaType.APPLICATION_JSON_VALUE)
21+
if (additionalHeaders.find { it.first.equals("Content-Type", ignoreCase = true) } == null)
22+
headers.add("Content-Type", MediaType.APPLICATION_JSON_VALUE)
23+
additionalHeaders.forEach { headers.add(it.first, it.second.toString()) }
2024
return headers
2125
}
2226

23-
fun empty(vararg additionalHeaders: Pair<String, Any?>): HttpEntity<Unit> {
24-
val headers = jsonHeaders()
25-
additionalHeaders.map { headers.add(it.first, it.second.toString()) }
26-
return HttpEntity(Unit, headers)
27-
}
27+
fun empty(vararg additionalHeaders: Pair<String, Any?>): HttpEntity<Unit> =
28+
HttpEntity(Unit, jsonHeaders(*additionalHeaders))
29+
30+
31+
fun <T> create(body: T, vararg additionalHeaders: Pair<String, Any?>): HttpEntity<T> =
32+
HttpEntity(body, jsonHeaders(*additionalHeaders))
2833

29-
fun <T> create(body: T, vararg additionalHeaders: Pair<String, Any?>): HttpEntity<T> {
30-
val headers = jsonHeaders()
31-
additionalHeaders.map { headers.add(it.first, it.second.toString()) }
32-
return HttpEntity(body, headers)
33-
}
3434
}
3535

3636

‎src/test/kotlin/ch/derlin/bbdata/input/InputApiTest.kt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ class InputApiTest {
142142

143143
@Test
144144
fun `1-2 test get latest measure`() {
145-
val (status, json) = restTemplate.getQueryJson("/objects/$OBJ/values/latest", "accept" to "application/json")
145+
val (status, json) = restTemplate.getQueryJson("/objects/$OBJ/values/latest")
146146
assertEquals(HttpStatus.OK, status)
147147

148148
val latestValue = json.read<Map<String, Any>>("$.[0]")
@@ -231,7 +231,7 @@ class InputApiTest {
231231
assertTrue(resp.body!!.contains("token"), "body should contain token related error ${resp.body}")
232232

233233
// assert the correct measures are not saved
234-
val (status, json) = restTemplate.getQueryJson("/objects/$OBJ/values?from=$now&to=$now", "accept" to "application/json")
234+
val (status, json) = restTemplate.getQueryJson("/objects/$OBJ/values?from=$now&to=$now")
235235
assertEquals(HttpStatus.OK, status, "get values $OBJ $now returned ${json.jsonString()}")
236236
assertEquals(0, json.read<List<Any>>("$[*]").size, "Failed bulk insert saved a value ! ${json.jsonString()}")
237237

@@ -250,7 +250,7 @@ class InputApiTest {
250250
assertEquals(HttpStatus.OK, resp.statusCode, "submit multiple measures returned ${resp.body}")
251251

252252
// assert the measures are saved
253-
val (status, json) = restTemplate.getQueryJson("/objects/$oid/values?from=$now&to=$now", "accept" to "application/json")
253+
val (status, json) = restTemplate.getQueryJson("/objects/$oid/values?from=$now&to=$now")
254254
assertEquals(HttpStatus.OK, status, "get values $oid $now returned ${json.jsonString()}")
255255
assertEquals(1, json.read<List<Any>>("$[*]").size, "bulk insert missing value ${json.jsonString()}")
256256

‎src/test/kotlin/ch/derlin/bbdata/output/TestStats.kt‎

Lines changed: 30 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -70,15 +70,15 @@ class TestStats {
7070
}
7171

7272
@Test
73-
fun `1-2 test one value`() {
73+
fun `1-2 test write one value`() {
7474
// post a new measure
7575
val json = submitAndCheck("12.2")
7676
assertEquals(.0, json.readPeriod(),
7777
"after submitting only one value, avgSamplePeriod should be 0")
7878
}
7979

8080
@Test
81-
fun `1-3 test another value`() {
81+
fun `1-3 test write another value`() {
8282
// post a new measure
8383
val json = submitAndCheck("6.")
8484
assertTrue(json.readPeriod() > .0,
@@ -90,7 +90,7 @@ class TestStats {
9090
val url = "/objects/$OID/values?from=${InputApiTest.tsFmt()}"
9191
val resp = restTemplate.getQueryString(url)
9292
assertEquals(HttpStatus.OK, resp.statusCode, "get $url returned ${resp.body}")
93-
nReads = +1
93+
nReads += 1
9494
submitAndCheck("6.")
9595

9696
}
@@ -103,6 +103,33 @@ class TestStats {
103103
assertNotEquals(HttpStatus.OK, status, "get $url returned ${json.jsonString()}")
104104
}
105105

106+
@Test
107+
fun `3-1 test read one value on a new object`() {
108+
// create a new object
109+
val oid = restTemplate.createObject(owner = REGULAR_USER_ID)
110+
111+
// read once
112+
var url = "/objects/$oid/values?from=2020-01-01&to=2020-01-02"
113+
restTemplate.getQueryJson(url).let { (status, json) ->
114+
assertEquals(HttpStatus.OK, status, "get $url returned ${json.jsonString()}")
115+
}
116+
117+
// get current stats
118+
url = "/objects/$oid/stats"
119+
val (status, json) = restTemplate.getQueryJson(url)
120+
assertEquals(HttpStatus.OK, status, "get $url returned ${json.jsonString()}")
121+
122+
listOf(
123+
"objectId" to oid,
124+
"nReads" to 1,
125+
"nWrites" to 0,
126+
"avgSamplePeriod" to .0,
127+
"lastTs" to null
128+
).forEach { (key, expected) ->
129+
assertEquals(expected, json.read("$.$key"), "get $url one read: should have $key=$expected, ${json.jsonString()}")
130+
}
131+
}
132+
106133
// ----------
107134

108135
private fun submitAndCheck(value: String): DocumentContext {

0 commit comments

Comments
 (0)