Skip to content

Commit 7c0d602

Browse files
karel-rehorkarel-rehor
committed
fix: (WIP) ensure tag keys and values with commas can be parsed correctly.
1 parent 70bd52e commit 7c0d602

2 files changed

Lines changed: 145 additions & 2 deletions

File tree

‎client/src/main/java/com/influxdb/client/internal/InfluxQLQueryApiImpl.java‎

Lines changed: 91 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,8 +120,14 @@ static InfluxQLQueryResult readInfluxQLResult(
120120
// All other columns are dynamically parsed
121121
final int dynamicColumnsStartIndex = 2;
122122

123-
try (CSVParser parser = new CSVParser(reader, CSVFormat.DEFAULT.builder().setIgnoreEmptyLines(false).build())) {
123+
// CSVFormat.RFC4180.builder()
124+
try (CSVParser parser = new CSVParser(reader, CSVFormat.DEFAULT.builder()
125+
// .setEscape('\\')
126+
.setIgnoreEmptyLines(false)
127+
// .setTrim(true)
128+
.build())) {
124129
for (CSVRecord csvRecord : parser) {
130+
// System.out.println("DEBUG csvRecord: " + csvRecord + ", size " + csvRecord.size());
125131
if (cancellable.isCancelled()) {
126132
break;
127133
}
@@ -156,8 +162,11 @@ static InfluxQLQueryResult readInfluxQLResult(
156162
Arrays.asList(name, finalTags),
157163
n -> new InfluxQLQueryResult.Series(name, finalTags, finalHeaderCols)
158164
);
165+
// System.out.println("DEBUG finalHeaderCols: " + finalHeaderCols + " dynamiColumnsStartIndex: " + dynamicColumnsStartIndex);
166+
//getCSVField(csvRecord, headerCols.get("time") + dynamicColumnsStartIndex);
159167
Object[] values = headerCols.entrySet().stream().map(entry -> {
160-
String value = csvRecord.get(entry.getValue() + dynamicColumnsStartIndex);
168+
// String value = csvRecord.get(entry.getValue() + dynamicColumnsStartIndex);
169+
String value = getCSVField(csvRecord, entry.getValue() + dynamicColumnsStartIndex);
161170
if (valueExtractor != null) {
162171
return valueExtractor.extractValue(entry.getKey(), value, resultIndex, serie.getName());
163172
}
@@ -178,6 +187,84 @@ static InfluxQLQueryResult readInfluxQLResult(
178187
return new InfluxQLQueryResult(results);
179188
}
180189

190+
// Need to fixup any fields that might have contained a commented comma
191+
private static String getCSVField(CSVRecord record, int col_index) {
192+
ArrayList<String> fixupValues = new ArrayList<>();
193+
StringBuilder sb = new StringBuilder();
194+
for (int i = 0; i < record.size(); i++) {
195+
// System.out.println("DEBUG record.values()["+ i + "]: " + record.values()[i]);
196+
if(record.values()[i].endsWith("\\")){
197+
sb.append(record.get(i)).append(",");
198+
} else {
199+
sb.append(record.get(i));
200+
fixupValues.add(sb.toString());
201+
sb.delete(0, sb.length());
202+
}
203+
}
204+
// System.out.println("DEBUG fixupValues: " + fixupValues);
205+
// System.out.println("DEBUG co_index: " + col_index + " fixupValues.size: " + fixupValues.size());
206+
// System.out.println("DEBUG result: " + fixupValues.get(col_index));
207+
return fixupValues.get(col_index);
208+
}
209+
210+
private static int IndexOfUnescapedChar(String str, char ch) {
211+
char[] chars = str.toCharArray();
212+
for (int i = 0; i < chars.length; i++) {
213+
if (chars[i] == ch && chars[i-1] != '\\') {
214+
return i;
215+
}
216+
}
217+
return -1;
218+
}
219+
220+
private static Map<String, String> parseTags(@Nonnull final String value) {
221+
final Map<String, String> tags = new HashMap<>();
222+
final List<String> keys = new ArrayList<>();
223+
final List<String> values = new ArrayList<>();
224+
// System.out.println("DEBUG ARG value: " + value);
225+
if (!value.isEmpty()) {
226+
String[] chunks = value.split("=");
227+
for (int i = 0; i < chunks.length; i++) {
228+
// System.out.println("DEBUG chunks[" + i + "]: " + chunks[i]);
229+
if (i == 0) {
230+
keys.add(chunks[i]);
231+
} else if (i == chunks.length - 1) {
232+
values.add(chunks[i]);
233+
} else {
234+
int comma_index = IndexOfUnescapedChar(chunks[i], ',');
235+
if (comma_index != -1) {
236+
String v = chunks[i].substring(0, comma_index);
237+
String k = chunks[i].substring(comma_index + 1);
238+
// System.out.println("DEBUG v: " + v + " k: " + k);
239+
keys.add(k);
240+
values.add(v);
241+
}
242+
}
243+
}
244+
// System.out.println("DEBUG keys: ");
245+
// for(String key : keys) {
246+
// System.out.println(" key: " + key);
247+
//}
248+
// System.out.println("DEBUG values: ");
249+
//for(String val : values) {
250+
// System.out.println(" val: " + val);
251+
//}
252+
for (int i = 0; i < keys.size(); i++) {
253+
// tags.put(keys.get(i), values.get(i));
254+
tags.put(
255+
keys.get(i).contains("\\,") ? "\"" + keys.get(i) + "\"" : keys.get(i),
256+
values.get(i).contains("\\,") ? "\"" + values.get(i) + "\"" : values.get(i)
257+
);
258+
}
259+
}
260+
// System.out.println("DEBUG tags: ");
261+
// for(String key : tags.keySet()) {
262+
// System.out.println(" tags[" + key + "]: " + tags.get(key));
263+
// }
264+
return tags;
265+
}
266+
267+
/*
181268
private static Map<String, String> parseTags(@Nonnull final String value) {
182269
final Map<String, String> tags = new HashMap<>();
183270
if (value.length() > 0) {
@@ -189,4 +276,6 @@ private static Map<String, String> parseTags(@Nonnull final String value) {
189276
190277
return tags;
191278
}
279+
280+
*/
192281
}

‎client/src/test/java/com/influxdb/client/internal/InfluxQLQueryApiImplTest.java‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,17 @@
2424
import java.io.IOException;
2525
import java.io.StringReader;
2626
import java.time.Instant;
27+
import java.util.ArrayList;
28+
import java.util.Arrays;
29+
import java.util.HashMap;
2730
import java.util.List;
31+
import java.util.Map;
2832

2933
import com.influxdb.Cancellable;
3034
import com.influxdb.query.InfluxQLQueryResult;
3135
import org.assertj.core.api.Assertions;
3236
import org.junit.jupiter.api.Test;
37+
import org.junit.platform.engine.TestTag;
3338

3439
class InfluxQLQueryApiImplTest {
3540

@@ -44,6 +49,55 @@ public boolean isCancelled() {
4449
}
4550
};
4651

52+
@Test
53+
void readInfluxQLResultWithTagCommas() throws IOException {
54+
InfluxQLQueryResult.Series.ValueExtractor extractValue = (columnName, rawValue, resultIndex, seriesName) -> {
55+
// System.out.println("DEBUG columnName: " + columnName + ", rawValue: " + rawValue + ", resultIndex: " + resultIndex + ", seriesName: " + seriesName);
56+
if (resultIndex == 0 && seriesName.equals("data1")){
57+
switch (columnName){
58+
case "time": return Instant.ofEpochSecond(Long.parseLong(rawValue));
59+
case "first": return Double.valueOf(rawValue);
60+
//case "tags": return rawValue;
61+
}
62+
}
63+
return rawValue;
64+
};
65+
66+
// Cheb,CZ should be \"Cheb,CZ\" a single tag value
67+
// double quotes should work - from raw sample results commas should always be escaped
68+
StringReader reader = new StringReader("name,tags,time,first\n"
69+
+ "data1,\"location=Cheb_CZ\",1483225200,42\n"
70+
+ "data1,\"region=us-east-1,host=server1\",1483225200,13.57\n"
71+
// + "data1,\"location=Cheb,CZ\",1483225200,42\n" // invalid - comma in value should be escaped
72+
// + "data1,\"location=Cheb, CZ\",1483225200,42\n" // invalid - comma and space in value should be escaped
73+
+ "data1,\"location=Cheb\\,\\ CZ\",1483225200,42\n"
74+
+ "data1,\"location=Cheb_CZ,branch=Munchen_DE\",1483225200,42\n"
75+
+ "data1,\"location=Cheb\\,\\ CZ,branch=Munchen\\,\\ DE\",1483225200,42\n"
76+
+ "data1,\"model\\,\\ uin=C3PO\",1483225200,42\n"
77+
+ "data1,\"model\\,\\ uin=Droid\\, C3PO\",1483225200,42\n"
78+
+ "data1,\"location=Cheb\\,\\ CZ,branch=Munchen\\,\\ DE\",1483225200,42\n"
79+
+ "data1,\"model\\,\\ uin=Droid\\,\\ C3PO,location=Cheb\\,\\ CZ,branch=Munchen\\,\\ DE\",1483225200,42\n"
80+
+ "data1,\"silly\\,long\\,tag=a\\,b\\,\\ c\\,\\ d\",1483225200,42\n"
81+
+ "\n"
82+
+ "name,tags,time,usage_user,usage_system\n"
83+
+ "cpu,\"region=us\\,\\ east-1,host\\,\\ name=ser\\,\\ ver1\",1483225200,13.57,1.4\n"
84+
);
85+
86+
// TODO meaningful asserts
87+
InfluxQLQueryResult result = InfluxQLQueryApiImpl.readInfluxQLResult(reader, NO_CANCELLING, extractValue);
88+
List<InfluxQLQueryResult.Result> results = result.getResults();
89+
// System.out.println("DEBUG results\n" + results.get(0).getSeries().get(0).getValues().get(0).getValueByKey("tags"));
90+
}
91+
92+
/*
93+
Sample response 1 - note escaped commas
94+
name,tags,time,fVal,iVal,id,location,"location\,boo",model,"model\,uin",sVal
95+
zaphrod_b,,1773307528202967039,26.54671,-6922649068284626682,bar,Harfa,,R2D2,,FOO
96+
zaphrod_b,,1773322199131651270,26.54671,-6922649068284626682,bar,,Harfa,R2D2,,FOO
97+
zaphrod_b,,1773322228235655514,26.54671,-6922649068284626682,bar,,"Harfa\,\ Praha",R2D2,,FOO
98+
zaphrod_b,,1773322254827374192,26.54671,-6922649068284626682,bar,,"Harfa\,\ Praha",,R2D2,FOO
99+
*/
100+
47101
@Test
48102
void readInfluxQLResult() throws IOException {
49103
InfluxQLQueryResult.Series.ValueExtractor extractValues = (columnName, rawValue, resultIndex, seriesName) -> {

0 commit comments

Comments
 (0)