From 9aa00c9e0e83c2a2cb60273b430a5cd0fbc34b61 Mon Sep 17 00:00:00 2001 From: eugenioseveri Date: Mon, 20 Jul 2026 17:02:42 +0200 Subject: [PATCH 1/3] Added CLAUDE.md --- CLAUDE.md | 96 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..e63c041 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,96 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Overview + +`routingfilter` is a Python library ("Generic Business Logic Implementation for Routing objects as python dictionaries") published to PyPI. Given a set of routing rules and an event (both plain `dict`s), it decides which rules an event matches and returns the associated outputs. It is a pure library with no runtime service — the source lives in `routingfilter/` and everything else (tests, benchmark, docs) is tooling around it. + +## Common commands + +All commands are run from the **repository root** (not from the inner `routingfilter/` package dir) — the test/benchmark code loads fixtures from `test_data/` via relative paths. + +```bash +# Install for development +pip install -r requirements.txt -r requirements_dev.txt +pre-commit install -c .github/.pre-commit-config.yaml + +# Run the full test suite +pytest routing_test.py + +# Run a single test +pytest routing_test.py::RoutingTestCase::test_multiple_rule_loading + +# Performance benchmark (not part of the unit tests) +python routing_benchmark.py + +# Linting (must pass in CI; configs live under .github/configurations/python_linters/) +black ./routingfilter --config .github/configurations/python_linters/.black --check --diff +flake8 ./routingfilter --config .github/configurations/python_linters/.flake8 --show-source +isort ./routingfilter --sp .github/configurations/python_linters/.isort.cfg --profile black --filter-files --check-only --diff +``` + +Notes: +- CI (`.github/workflows/python-app.yml`) only runs the build/lint/test job when files under `routingfilter/*` change. +- Line length is 160 across black/flake8/isort. + +## Architecture + +The matching engine is a strict containment hierarchy. Loading rules builds the tree top-down; matching an event walks it top-down and short-circuits. + +``` +Routing # entry point (routing.py) + ├─ streams: Stream # the "streams" ruleset + └─ customer: Stream # the "customers" ruleset + └─ RuleManager # one per tag (keyed in Stream._ruleManagers by tag string) + └─ Rule # has an output + a list of filters (AND semantics) + └─ AbstractFilter subclasses (filters/filters.py) +``` + +- **`Routing`** (`routingfilter/routing.py`) is the public API. `load_from_dicts()` / `load_from_jsons()` parse rule configs and instantiate the whole tree; `_get_filters()` is the central factory mapping a filter `type` string (e.g. `"EQUALS"`, `"NETWORK"`) to a concrete filter class. `match(event, type_="streams", tag_field_name="tags")` dispatches to the `streams` or `customer` Stream and returns a `List[Results]`. + +- **`Stream`** (`filters/stream.py`) holds `RuleManager`s keyed by tag. On `match()` it reads the event's tags (from `tag_field_name`, default `"tags"`) and invokes only the matching-tag RuleManagers. The special tag **`"all"`** is checked first and short-circuits: if an `all` RuleManager matches, its single result is returned and no other tags are evaluated. + +- **`RuleManager`** (`filters/rule.py`) owns an ordered list of `Rule`s for one tag and returns the **first** matching rule (OR / priority-order semantics). + +- **`Rule`** (`filters/rule.py`) matches only if **all** its filters match (AND). On a successful match it records the output keys with a timestamp into the event's `certego.routing_history`, which prevents the same output key from being emitted twice across repeated matching. It also tracks per-rule hit stats keyed by the event's `rule.name`. + +- **Filters** (`filters/filters.py`) all subclass `AbstractFilter` and implement `match(event) -> bool` plus `_check_value()` (validates/normalizes the configured values, e.g. lowercasing, compiling regexes, parsing IPs; raises `ValueError` on bad config). Available types: `ALL`, `EXISTS`, `NOT_EXISTS`, `EQUALS`, `NOT_EQUALS`, `STARTSWITH`, `ENDSWITH`, `KEYWORD`, `REGEXP`, `NETWORK`, `NOT_NETWORK`, `DOMAIN`, `GREATER`/`LESS`/`GREATER_EQ`/`LESS_EQ` (all `ComparatorFilter`), `TYPEOF`. Most string comparisons are **case-insensitive** (values are lowercased in `_check_value`). + +- **`DictQuery`** (`routingfilter/dictquery.py`) is a `dict` subclass whose `get()` walks dotted paths (`"source.ip"`). A literal key containing a `.` is matched before the path is split. This is how filter `key` fields address nested event fields. + +- **`Results`** (`filters/results.py`) is the `dataclass` returned for each match (`{rules, output}`). If an output dict contains a `"customer"` key, its value is unwrapped as the output. + +### Rule config shape + +Rules are dicts nested by stream type → `"rules"` → tag → list of rule objects. Each rule object has a `filters` list, an optional output under the stream-type key, and an optional `id` (a UUID is generated if absent). Example: + +```json +{ + "streams": { + "rules": { + "my_tag": [ + { + "filters": [{"type": "EQUALS", "key": "source.ip", "value": "1.2.3.4"}], + "streams": {"...output..."}, + "id": "optional-id" + } + ] + } + } +} +``` + +`load_from_dicts(..., variables={...})` supports variable substitution: filter values that are variable names (referenced with a `$` prefix) are replaced by their configured values before filters are built (`Routing._substitute_variables`). + +Extensive real examples of both rules and events live in `test_data/` (`test_rule_*.json`, `test_event_*.json`), which is the best reference when writing or debugging rules. + +## Adding a new filter type + +1. Add an `AbstractFilter` subclass in `filters/filters.py` implementing `match()` and `_check_value()`. +2. Register its `type` string in the `match` statement of `Routing._get_filters()` (`routing.py`). +3. Add a rule fixture in `test_data/` and a corresponding test case in `routing_test.py`. + +## Release process + +Update `requirements.txt`/`setup.py` if needed, add a `CHANGELOG.md` entry, bump the version in `setup.py`, merge to `master`, then publish a GitHub release tagged with the version — CI (`python-publish.yml`) publishes to PyPI automatically. \ No newline at end of file From caec293f7e07f1b3817b7077bde36d88a4037db7 Mon Sep 17 00:00:00 2001 From: eugenioseveri Date: Mon, 20 Jul 2026 17:03:42 +0200 Subject: [PATCH 2/3] Updated CI and fixed CI bugs --- .github/workflows/python-app.yml | 17 +++++++++-------- .github/workflows/python-publish.yml | 6 +++--- .idea/misc.xml | 3 +++ CHANGELOG.md | 6 ++++++ 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index e1e6e1d..f155215 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -12,14 +12,14 @@ on: jobs: detect-changes: name: Detect changes - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: python_code: ${{steps.diff_check.outputs.python_code}} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v7 with: ref: ${{ github.base_ref }} - - uses: actions/checkout@v3 + - uses: actions/checkout@v7 with: clean: false - name: Generate diffs @@ -27,22 +27,23 @@ jobs: run: | git branch -a --list | cat PYTHON_CODE_CHANGES=$(git diff --compact-summary origin/${{ github.base_ref }} -- routingfilter/* | wc -l) - echo "::set-output name=python_code::$PYTHON_CODE_CHANGES" + echo "python_code=$PYTHON_CODE_CHANGES" >> "$GITHUB_OUTPUT" build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: [ "detect-changes" ] if: ${{ needs.detect-changes.outputs.python_code > 0 }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v7 - name: Set up Python - uses: actions/setup-python@v4 + id: setup_python + uses: actions/setup-python@v7 with: python-version: "3.10" - name: "Cache venv" id: cache_venv - uses: actions/cache@v3 + uses: actions/cache@v6 with: path: venv key: pip-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('requirements.txt') }} diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 00e5bc1..c881efd 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -18,9 +18,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v7 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v7 with: python-version: '3.x' - name: Install dependencies @@ -30,7 +30,7 @@ jobs: - name: Build package run: python -m build - name: Publish package - uses: pypa/gh-action-pypi-publish@v1.8.9 + uses: pypa/gh-action-pypi-publish@v1.14.1 with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.idea/misc.xml b/.idea/misc.xml index f58ef9e..50fcc30 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,4 +1,7 @@ + + \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index fadeed5..32f92a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 2.4.x +### 2.4.0 +#### Bugfix +* Fixed bug in CI about isort execution and cache +#### Changes +* Updated linters, required Python version (3.10) and Github Actions ## 2.3.x ### 2.3.3 #### Changes From 4667eeb82a856fdc653399521a3cd6c2118f9807 Mon Sep 17 00:00:00 2001 From: eugenioseveri Date: Mon, 20 Jul 2026 17:04:24 +0200 Subject: [PATCH 3/3] Removed dependency from `IPy` in favor of `ipdaddress` --- CHANGELOG.md | 3 ++- requirements.txt | 1 - routing_test.py | 4 ++-- routingfilter/filters/filters.py | 11 ++++++----- setup.py | 4 ++-- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32f92a0..37d421b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,8 @@ #### Bugfix * Fixed bug in CI about isort execution and cache #### Changes -* Updated linters, required Python version (3.10) and Github Actions +* Updated linters, required Python version (>3.10) and Github Actions +* Removed dependency from `IPy` in favor of `ipdaddress` ## 2.3.x ### 2.3.3 #### Changes diff --git a/requirements.txt b/requirements.txt index 4194d8d..e8eb67e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1 @@ -IPy~=1.1 macaddress~=2.0.2 \ No newline at end of file diff --git a/routing_test.py b/routing_test.py index 6b92336..24df06a 100644 --- a/routing_test.py +++ b/routing_test.py @@ -1,9 +1,9 @@ import copy +import ipaddress import json import os import unittest -from IPy import IP from routingfilter.filters import filters from routingfilter.routing import Routing @@ -390,7 +390,7 @@ def test_multiple_variables_list(self): self.routing.load_from_dicts([load_test_data("test_rule_33_network_multiple_variables")], variables={"$HOME_NET": ["192.168.1.0/24"]}) self.assertDictEqual(self.routing.variables, {"$HOME_NET": ["192.168.1.0/24"]}) values = self.routing.streams._ruleManagers["ip_traffic"]._rules[0]._filters[0]._value - self.assertEqual([IP("192.168.1.0/24"), IP("10.0.0.1")], values) + self.assertEqual([ipaddress.ip_network("192.168.1.0/24"), ipaddress.ip_network("10.0.0.1")], values) self.assertTrue(self.routing.match(self.test_event_4)) def test_rule_upper_case_value(self): diff --git a/routingfilter/filters/filters.py b/routingfilter/filters/filters.py index df58736..eb8d559 100644 --- a/routingfilter/filters/filters.py +++ b/routingfilter/filters/filters.py @@ -1,10 +1,10 @@ +import ipaddress import logging import re from abc import ABC, abstractmethod from typing import NoReturn, Optional import macaddress -from IPy import IP from routingfilter.dictquery import DictQuery @@ -315,7 +315,7 @@ def _check_value(self) -> Exception | NoReturn: tmp = [] for value in self._value: try: - value = IP(value) + value = ipaddress.ip_network(value) except ValueError as e: self.logger.error(f"IP address (value error) error, during check of value {value} in list {self._value}. Error was: {e}.") raise ValueError(f"IP address check failed: value error for value {value}.") @@ -352,9 +352,10 @@ def _check_network(self, ip_address: str) -> bool: :rtype: bool """ try: - ip_address = IP(ip_address) + network = ipaddress.ip_network(ip_address) for value in self._value: - if ip_address in value: + # Also check the IP protocol version because "supernet_of" raises a TypeError when comparing IPv4 and IPv6 addresses + if network.version == value.version and value.supernet_of(network): return True except ValueError as e: self.logger.debug(f"Error in parsing IP address (value error): {e}. ") @@ -588,7 +589,7 @@ def _check_ip(self, value: any) -> bool: return False except ValueError: try: - IP(value) + ipaddress.ip_address(value) return True except ValueError: return False diff --git a/setup.py b/setup.py index ebaca87..75ae195 100644 --- a/setup.py +++ b/setup.py @@ -5,10 +5,10 @@ setup( name="routingfilter", - version="2.3.3", + version="2.4.0", packages=find_packages(include=["routingfilter", "routingfilter.*"]), include_package_data=True, - install_requires=["IPy~=1.1", "macaddress~=2.0.2"], + install_requires=["macaddress~=2.0.2"], url="https://github.com/certego/RoutingFilter", license="GNU LGPLv3", author="Certego S.r.l.",