Skip to content

Commit a079144

Browse files
Lukas955Lukas955
committed
JSON converter: add support for ignoring reverse fields, unit tests
1 parent eff6dfd commit a079144

3 files changed

Lines changed: 118 additions & 25 deletions

File tree

‎include/libfds/converters.h‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1367,36 +1367,42 @@ enum fds_convert_drec2json {
13671367
* By default, it is interpreted from forward point of view.
13681368
*/
13691369
FDS_CD2J_BIFLOW_REVERSE = (1U << 1),
1370+
/**
1371+
* In case of a Biflow record, skip (i.e. ignore) all reverse fields.
1372+
* The flag can be combined with ::FDS_CD2J_BIFLOW_REVERSE. In that case,
1373+
* the filter is applied on the remapped fields.
1374+
*/
1375+
FDS_CD2J_REVERSE_SKIP = (1U << 2),
13701376
/**
13711377
* Skip fields with unknown definition of an Information Element.
13721378
*/
1373-
FDS_CD2J_IGNORE_UNKNOWN = (1U << 2),
1379+
FDS_CD2J_IGNORE_UNKNOWN = (1U << 3),
13741380
/**
13751381
* Convert standard TCP flags identification (i.e. "iana:tcpControlBits")
13761382
* to textual representation. For example, ".A..S.".
13771383
*/
1378-
FDS_CD2J_FORMAT_TCPFLAGS = (1U << 3),
1384+
FDS_CD2J_FORMAT_TCPFLAGS = (1U << 4),
13791385
/**
13801386
* Convert standart protocol identification (i.e. "iana:protocolIdentifier")
13811387
* to textual representation. For example, instead of 6 writes "TCP").
13821388
*/
1383-
FDS_CD2J_FORMAT_PROTO = (1U << 4),
1389+
FDS_CD2J_FORMAT_PROTO = (1U << 5),
13841390
/**
13851391
* Ingore non-printable charactes (newline, tab, control charactes, etc.)
13861392
* in IPFIX string fields instead of escaping them.
13871393
*/
1388-
FDS_CD2J_NON_PRINTABLE = (1U << 5),
1394+
FDS_CD2J_NON_PRINTABLE = (1U << 6),
13891395
/**
13901396
* Use the format of unknown Information Elements (i.e. "enXX:idYY") for
13911397
* names for all name-value pairs. E.g., instead of "iana:octetDeltaCount"
13921398
* use always "en0:id1".
13931399
*/
1394-
FDS_CD2J_NUMERIC_ID = (1U << 6),
1400+
FDS_CD2J_NUMERIC_ID = (1U << 7),
13951401
/**
13961402
* Convert all timestamps to ISO 8601 textual representation of UTC with
13971403
* milliseconds, e.g. "2019-05-22T22:34:57.828Z"
13981404
*/
1399-
FDS_CD2J_TS_FORMAT_MSEC = (1U << 7),
1405+
FDS_CD2J_TS_FORMAT_MSEC = (1U << 8)
14001406
};
14011407

14021408
/**

‎src/converters/json.c‎

Lines changed: 15 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -836,24 +836,21 @@ to_proto(struct context *buffer, const struct fds_drec_field *field)
836836
* \breaf Auxiliary function for converting record fields with multiple occurrence
837837
*
838838
* The values are stored into a simple JSON array identified by the field ID.
839-
* \param[in] rec IPFIX Data Record with the fields
840-
* \param[in] buffer Buffer
841-
* \param[in] fn Convert function applied on the values
842-
* \param[in] en Enterprise Number of the field
843-
* \param[in] id Information Element ID of the field
839+
* \param[in] rec IPFIX Data Record with the fields
840+
* \param[in] buffer Buffer
841+
* \param[in] fn Convert function applied on the values
842+
* \param[in] en Enterprise Number of the field
843+
* \param[in] id Information Element ID of the field
844+
* \param[in] iter_flag Data Record iterator flags
844845
*
845846
* \return #FDS_OK on success
846847
* \return #FDS_ERR_NOMEM or #FDS_ERR_BUFFER in case of a memory allocation error
847-
*
848-
*/
848+
*/
849849
static int
850850
multi_fields(const struct fds_drec *rec, struct context *buffer,
851-
converter_fn fn, uint32_t en, uint16_t id)
851+
converter_fn fn, uint32_t en, uint16_t id, uint16_t iter_flag)
852852
{
853-
// Inicialization of iterator
854-
uint16_t iter_flag = (buffer->flags & FDS_CD2J_IGNORE_UNKNOWN) ? FDS_DREC_UNKNOWN_SKIP : 0;
855-
iter_flag |= (buffer->flags & FDS_CD2J_BIFLOW_REVERSE) ? FDS_DREC_BIFLOW_REV : 0;
856-
853+
// Inicialization of an iterator
857854
struct fds_drec_iter iter_mul_f;
858855
fds_drec_iter_init(&iter_mul_f, (struct fds_drec *) rec, iter_flag);
859856

@@ -986,12 +983,11 @@ iter_loop(const struct fds_drec *rec, struct context *buffer)
986983
unsigned int added = 0;
987984
int ret_code;
988985

989-
// Try to convert each field in the record
990-
uint16_t iter_flag = (buffer->flags & FDS_CD2J_IGNORE_UNKNOWN) ? FDS_DREC_UNKNOWN_SKIP : 0;
991-
992-
// If flag FDS_CD2J_BIFLOW_REVERSE is set,
993-
// then will be added flag FDS_DREC_BIFLOW_REV for every field
994-
iter_flag |= (buffer->flags & FDS_CD2J_BIFLOW_REVERSE) ? FDS_DREC_BIFLOW_REV : 0;
986+
// Initialize iterator flags
987+
uint16_t iter_flag = 0;
988+
iter_flag |= (buffer->flags & FDS_CD2J_IGNORE_UNKNOWN) ? FDS_DREC_UNKNOWN_SKIP : 0;
989+
iter_flag |= (buffer->flags & FDS_CD2J_BIFLOW_REVERSE) ? FDS_DREC_BIFLOW_REV : 0;
990+
iter_flag |= (buffer->flags & FDS_CD2J_REVERSE_SKIP) ? FDS_DREC_REVERSE_SKIP : 0;
995991

996992
struct fds_drec_iter iter;
997993
fds_drec_iter_init(&iter, (struct fds_drec *) rec, iter_flag);
@@ -1038,7 +1034,7 @@ iter_loop(const struct fds_drec *rec, struct context *buffer)
10381034
char *writer_pos = buffer->write_begin;
10391035
if ((field_flags & FDS_TFIELD_MULTI_IE) != 0 && (field_flags & FDS_TFIELD_LAST_IE) != 0) {
10401036
// Conversion of the field with multiple occurrences
1041-
ret_code = multi_fields(rec, buffer, fn, def->en, def->id);
1037+
ret_code = multi_fields(rec, buffer, fn, def->en, def->id, iter_flag);
10421038
if (ret_code != FDS_OK) {
10431039
return ret_code;
10441040
}

‎tests/unit_tests/converters/json.cpp‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -359,6 +359,97 @@ TEST_F(Drec_biflow, simpleParser)
359359
free(buff);
360360
}
361361

362+
// Convert only forward fields
363+
TEST_F(Drec_biflow, forwardOnly)
364+
{
365+
constexpr size_t BSIZE = 2U;
366+
char* buff = (char*) malloc(BSIZE);
367+
uint32_t flags = FDS_CD2J_ALLOW_REALLOC | FDS_CD2J_REVERSE_SKIP;
368+
size_t buff_size = BSIZE;
369+
370+
int rc = fds_drec2json(&m_drec, flags, m_iemgr.get(), &buff, &buff_size);
371+
ASSERT_GT(rc, 0);
372+
EXPECT_EQ(size_t(rc), strlen(buff));
373+
EXPECT_NE(buff_size, BSIZE);
374+
Config cfg = parse_string(buff, JSON, "drec2json");
375+
EXPECT_TRUE(cfg.has_key("iana:sourceTransportPort"));
376+
EXPECT_TRUE(cfg.has_key("iana:sourceIPv6Address"));
377+
EXPECT_TRUE(cfg.has_key("iana:destinationTransportPort"));
378+
EXPECT_TRUE(cfg.has_key("iana:destinationIPv6Address"));
379+
EXPECT_TRUE(cfg.has_key("iana:protocolIdentifier"));
380+
EXPECT_TRUE(cfg.has_key("iana:flowStartNanoseconds"));
381+
EXPECT_TRUE(cfg.has_key("iana:flowEndNanoseconds"));
382+
EXPECT_TRUE(cfg.has_key("iana:applicationName"));
383+
EXPECT_TRUE(cfg.has_key("iana:applicationDescription"));
384+
EXPECT_TRUE(cfg.has_key("iana:interfaceName"));
385+
EXPECT_TRUE(cfg.has_key("iana:octetDeltaCount"));
386+
EXPECT_TRUE(cfg.has_key("iana:packetDeltaCount"));
387+
388+
EXPECT_FALSE(cfg.has_key("iana@reverse:flowStartNanoseconds@reverse"));
389+
EXPECT_FALSE(cfg.has_key("iana@reverse:flowEndNanoseconds@reverse"));
390+
EXPECT_FALSE(cfg.has_key("iana@reverse:octetDeltaCount@reverse"));
391+
EXPECT_FALSE(cfg.has_key("iana@reverse:packetDeltaCount@reverse"));
392+
393+
EXPECT_EQ((uint64_t) cfg["iana:octetDeltaCount"], VALUE_BYTES); // octetDeltaCount
394+
EXPECT_EQ((uint64_t) cfg["iana:packetDeltaCount"], VALUE_PKTS); // packetDeltaCount
395+
EXPECT_EQ((uint64_t) cfg["iana:sourceTransportPort"], VALUE_SRC_PORT); // sourceTransportPort
396+
EXPECT_EQ( cfg["iana:sourceIPv6Address"], VALUE_SRC_IP6); // sourceIPv6Address
397+
EXPECT_EQ((uint64_t) cfg["iana:destinationTransportPort"], VALUE_DST_PORT); // destinationTransportPort
398+
EXPECT_EQ( cfg["iana:destinationIPv6Address"], VALUE_DST_IP6); // destinationIPv6Address
399+
EXPECT_EQ((uint64_t) cfg["iana:protocolIdentifier"], VALUE_PROTO); // protocolIdentifier
400+
EXPECT_EQ((uint64_t) cfg["iana:flowStartNanoseconds"], VALUE_TS_FST); // flowStartNanoseconds
401+
EXPECT_EQ((uint64_t) cfg["iana:flowEndNanoseconds"], VALUE_TS_LST); // flowEndNanoseconds
402+
EXPECT_EQ(cfg["iana:applicationName"], VALUE_APP_NAME); // applicationName
403+
EXPECT_EQ(cfg["iana:applicationDescription"], VALUE_APP_DSC); // applicationDescription
404+
free(buff);
405+
}
406+
407+
// Convert from reverse point of view without "forward only" fields
408+
TEST_F(Drec_biflow, ReverseOnly)
409+
{
410+
constexpr size_t BSIZE = 2U;
411+
char* buff = (char*) malloc(BSIZE);
412+
uint32_t flags = FDS_CD2J_ALLOW_REALLOC | FDS_CD2J_BIFLOW_REVERSE | FDS_CD2J_REVERSE_SKIP;
413+
size_t buff_size = BSIZE;
414+
415+
int rc = fds_drec2json(&m_drec, flags, m_iemgr.get(), &buff, &buff_size);
416+
ASSERT_GT(rc, 0);
417+
EXPECT_EQ(size_t(rc), strlen(buff));
418+
EXPECT_NE(buff_size, BSIZE);
419+
Config cfg = parse_string(buff, JSON, "drec2json");
420+
EXPECT_TRUE(cfg.has_key("iana:sourceTransportPort"));
421+
EXPECT_TRUE(cfg.has_key("iana:sourceIPv6Address"));
422+
EXPECT_TRUE(cfg.has_key("iana:destinationTransportPort"));
423+
EXPECT_TRUE(cfg.has_key("iana:destinationIPv6Address"));
424+
EXPECT_TRUE(cfg.has_key("iana:protocolIdentifier"));
425+
EXPECT_TRUE(cfg.has_key("iana:flowStartNanoseconds"));
426+
EXPECT_TRUE(cfg.has_key("iana:flowEndNanoseconds"));
427+
EXPECT_TRUE(cfg.has_key("iana:applicationName"));
428+
EXPECT_TRUE(cfg.has_key("iana:applicationDescription"));
429+
EXPECT_TRUE(cfg.has_key("iana:interfaceName"));
430+
EXPECT_TRUE(cfg.has_key("iana:octetDeltaCount"));
431+
EXPECT_TRUE(cfg.has_key("iana:packetDeltaCount"));
432+
433+
EXPECT_FALSE(cfg.has_key("iana@reverse:flowStartNanoseconds@reverse"));
434+
EXPECT_FALSE(cfg.has_key("iana@reverse:flowEndNanoseconds@reverse"));
435+
EXPECT_FALSE(cfg.has_key("iana@reverse:octetDeltaCount@reverse"));
436+
EXPECT_FALSE(cfg.has_key("iana@reverse:packetDeltaCount@reverse"));
437+
438+
// Source and destination fields must be swapped
439+
EXPECT_EQ((uint64_t) cfg["iana:octetDeltaCount"], VALUE_BYTES_R); // octetDeltaCount
440+
EXPECT_EQ((uint64_t) cfg["iana:packetDeltaCount"], VALUE_PKTS_R); // packetDeltaCount
441+
EXPECT_EQ((uint64_t) cfg["iana:sourceTransportPort"], VALUE_DST_PORT); // sourceTransportPort
442+
EXPECT_EQ( cfg["iana:sourceIPv6Address"], VALUE_DST_IP6); // sourceIPv6Address
443+
EXPECT_EQ((uint64_t) cfg["iana:destinationTransportPort"], VALUE_SRC_PORT); // destinationTransportPort
444+
EXPECT_EQ( cfg["iana:destinationIPv6Address"], VALUE_SRC_IP6); // destinationIPv6Address
445+
EXPECT_EQ((uint64_t) cfg["iana:protocolIdentifier"], VALUE_PROTO); // protocolIdentifier
446+
EXPECT_EQ((uint64_t) cfg["iana:flowStartNanoseconds"], VALUE_TS_FST_R); // flowStartNanoseconds
447+
EXPECT_EQ((uint64_t) cfg["iana:flowEndNanoseconds"], VALUE_TS_LST_R); // flowEndNanoseconds
448+
EXPECT_EQ(cfg["iana:applicationName"], VALUE_APP_NAME); // applicationName
449+
EXPECT_EQ(cfg["iana:applicationDescription"], VALUE_APP_DSC); // applicationDescription
450+
free(buff);
451+
}
452+
362453
// Convert Data Record with flag FDS_CD2J_NUMERIC_ID
363454
TEST_F(Drec_biflow, numID)
364455
{

0 commit comments

Comments
 (0)