Skip to content

Commit 9dab8df

Browse files
authored
Merge pull request #65 from zmap/phillip/63-greynoise-psychic-integration
Add GreyNoise Psychic data feed IP enrichment
2 parents e159cd5 + 9e1ab42 commit 9dab8df

4 files changed

Lines changed: 251 additions & 0 deletions

File tree

README.md

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,46 @@ Example Output:
140140
{"ip":"1.1.1.1","spur":{"as":{"number":13335,"organization":"Cloudflare, Inc."},"infrastructure":"DATACENTER","ip":"1.1.1.1","location":{"city":"Anycast","country":"ZZ","state":"Anycast"},"organization":"Taguchi Digital Marketing System"}}
141141
```
142142

143+
### Greynoise Psychic
144+
[Greynoise](https://greynoise.io) is an IP intelligence feed that provides metadata like threat classification and associated CVE's.
145+
Their Psychic data downloads provide their data feed in a database suitable for offline data enrichment.
146+
To use their download with `zannotate`, you'll want to download an `.mmdb` formatted file using your GreyNoise API key.
147+
As of April 2026, signing up with a free account gives access to data downloads.
148+
149+
0. Sign up for a free GreyNoise account [here](https://www.greynoise.io).
150+
1. Copy API key from the appropriate [section of your account](https://viz.greynoise.io/workspace/api-key).
151+
2. Download a `mmdb` file. Details on download parameters
152+
(The below command is for downloading data for a single date - April 7th, 2026 - you can also download data for a range of days and for models of various levels of detail.
153+
See GreyNoise's Psychic [documentation](https://psychic.labs.greynoise.io) for more details.
154+
```shell
155+
curl -H "key: GREYNOISE_API_KEY_HERE" \
156+
https://psychic.labs.greynoise.io/v1/psychic/download/2026-04-07/3/mmdb \
157+
-o /tmp/m3.mmdb
158+
```
159+
160+
3. Test GreyNoise data enrichment:
161+
162+
> [!NOTE]
163+
> The below examples are using the exact data download from the above `curl` command. What results you see will depend on the data downloaded.
164+
165+
```shell
166+
echo "14.1.105.157" | zannotate --greynoise --greynoise-database=/tmp/m3.mmdb
167+
````
168+
Example Output:
169+
```json
170+
{"greynoise":{"classification":"malicious","cves":["CVE-2015-2051","CVE-2016-20016","CVE-2018-10561","CVE-2018-10562","CVE-2016-6277","CVE-2024-12847"],"date":"2026-04-07","handshake_complete":true,"last_seen":"2026-04-07T00:00:00Z","seen":true,"tags":["Mirai TCP Scanner","Mirai","Telnet Protocol","Generic IoT Default Password Attempt","Web Crawler","Generic Suspicious Linux Command in Request","HNAP Crawler","Telnet Login Attempt","D-Link Devices HNAP SOAPAction Header RCE Attempt","MVPower CCTV DVR RCE CVE-2016-20016 Attempt","JAWS Webserver RCE","GPON CVE-2018-10561 Router Worm","Generic ${IFS} Use in RCE Attempt","CCTV-DVR RCE","NETGEAR Command Injection CVE-2016-6277","NETGEAR DGN setup.cgi CVE-2024-12847 Command Execution Attempt","CGI Script Scanner"],"actor":"unknown"},"ip":"14.1.105.157"}
171+
```
172+
173+
Note that many IPs will not be in the GreyNoise dataset, so you may see output like the following:
174+
```shell
175+
echo "1.1.1.1" | zannotate --greynoise --greynoise-database=/tmp/m3.mmdb
176+
```
177+
178+
```json
179+
{"ip":"1.1.1.1","greynoise":null}
180+
```
181+
182+
143183
# Input/Output
144184

145185
## Output

data-snapshots/greynoise.mmdb

15.8 MB
Binary file not shown.

greynoise_psychic.go

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
/*
2+
* ZAnnotate Copyright 2026 Regents of the University of Michigan
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy
6+
* of the License at http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
11+
* implied. See the License for the specific language governing
12+
* permissions and limitations under the License.
13+
*/
14+
15+
package zannotate
16+
17+
import (
18+
"errors"
19+
"flag"
20+
"fmt"
21+
"net"
22+
"net/netip"
23+
"os"
24+
25+
"github.com/oschwald/maxminddb-golang/v2"
26+
log "github.com/sirupsen/logrus"
27+
)
28+
29+
type GreyNoiseAnnotatorFactory struct {
30+
BasePluginConf
31+
DBPath string // path to the .mmdb path
32+
greynoiseDB *maxminddb.Reader
33+
}
34+
35+
// GreyNoise Annotator Factory (Global)
36+
37+
func (a *GreyNoiseAnnotatorFactory) MakeAnnotator(i int) Annotator {
38+
var v GreyNoiseAnnotator
39+
v.Factory = a
40+
v.Id = i
41+
return &v
42+
}
43+
44+
func (a *GreyNoiseAnnotatorFactory) Initialize(_ *GlobalConf) error {
45+
if len(a.DBPath) == 0 {
46+
return errors.New("greynoise database path is required when greynoise annotator is enabled, use --greynoise-database")
47+
}
48+
data, err := os.ReadFile(a.DBPath) // ensure DB is read in-memory
49+
if err != nil {
50+
return fmt.Errorf("unable to read greynoise database at %s: %w", a.DBPath, err)
51+
}
52+
a.greynoiseDB, err = maxminddb.OpenBytes(data)
53+
if err != nil {
54+
return fmt.Errorf("unable to open greynoise database at %s: %w", a.DBPath, err)
55+
}
56+
return nil
57+
}
58+
59+
func (a *GreyNoiseAnnotatorFactory) GetWorkers() int {
60+
return a.Threads
61+
}
62+
63+
func (a *GreyNoiseAnnotatorFactory) Close() error {
64+
if err := a.greynoiseDB.Close(); err != nil {
65+
return fmt.Errorf("unable to close greynoise database at %s: %w", a.DBPath, err)
66+
}
67+
return nil
68+
}
69+
70+
func (a *GreyNoiseAnnotatorFactory) IsEnabled() bool {
71+
return a.Enabled
72+
}
73+
74+
func (a *GreyNoiseAnnotatorFactory) AddFlags(flags *flag.FlagSet) {
75+
// Reverse DNS Lookup
76+
flags.BoolVar(&a.Enabled, "greynoise", false, "greynoise psychic data intelligence")
77+
flags.StringVar(&a.DBPath, "greynoise-database", "", "path to greynoise psychic .mmdb file")
78+
flags.IntVar(&a.Threads, "greynoise-threads", 2, "how many enrichment threads to use")
79+
}
80+
81+
// GreyNoiseAnnotator (Per-Worker)
82+
type GreyNoiseAnnotator struct {
83+
Factory *GreyNoiseAnnotatorFactory
84+
Id int
85+
}
86+
87+
func (a *GreyNoiseAnnotator) Initialize() (err error) {
88+
return nil
89+
}
90+
91+
func (a *GreyNoiseAnnotator) GetFieldName() string {
92+
return "greynoise"
93+
}
94+
95+
// Annotate performs a reverse DNS lookup for the given IP address and returns the results.
96+
// If an error occurs or a lookup fails, it returns nil
97+
func (a *GreyNoiseAnnotator) Annotate(ip net.IP) interface{} {
98+
addr, ok := netip.AddrFromSlice(ip)
99+
if !ok {
100+
log.Debugf("unable to convert IP %s to address", ip)
101+
return nil
102+
}
103+
addr = addr.Unmap()
104+
var result any
105+
err := a.Factory.greynoiseDB.Lookup(addr).Decode(&result)
106+
if err != nil {
107+
log.Debugf("unable to annotate IP (%s): %v", addr, err)
108+
return nil
109+
}
110+
return result
111+
}
112+
113+
func (a *GreyNoiseAnnotator) Close() error {
114+
return nil
115+
}
116+
117+
func init() {
118+
s := new(GreyNoiseAnnotatorFactory)
119+
RegisterAnnotator(s)
120+
}

greynoise_psychic_test.go

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
/*
2+
* ZAnnotate Copyright 2025 Regents of the University of Michigan
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy
6+
* of the License at http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
11+
* implied. See the License for the specific language governing
12+
* permissions and limitations under the License.
13+
*/
14+
package zannotate
15+
16+
import (
17+
"math/rand/v2"
18+
"net"
19+
"reflect"
20+
"testing"
21+
)
22+
23+
// TestGreyNoiseAnnotator tests that given a database file and a known IP, the annotator returns the expected values for that IP in the DB
24+
func TestGreyNoiseAnnotator(t *testing.T) {
25+
expectedFields := map[string]any{
26+
"actor": "Stanford University",
27+
"classification": "benign",
28+
"date": "2026-04-07",
29+
"handshake_complete": true,
30+
"last_seen": "2026-04-07T00:00:00Z",
31+
"seen": true,
32+
"tags": []any{"Stanford University", "RDP Crawler", "RDP Protocol"},
33+
}
34+
factory := &GreyNoiseAnnotatorFactory{DBPath: "./data-snapshots/greynoise.mmdb"}
35+
err := factory.Initialize(nil)
36+
if err != nil {
37+
t.Fatalf("Error initializing greynoise annotator factory: %v", err)
38+
}
39+
a := factory.MakeAnnotator(0).(*GreyNoiseAnnotator)
40+
err = a.Initialize()
41+
if err != nil {
42+
t.Fatalf("Error initializing greynoise annotator: %v", err)
43+
}
44+
45+
ip := "171.67.71.209"
46+
res := a.Annotate(net.ParseIP(ip))
47+
if res == nil {
48+
t.Fatalf("GreyNoiseAnnotator failed to annotate %s", ip)
49+
}
50+
51+
for field, expected := range expectedFields {
52+
actual, ok := res.(map[string]any)[field]
53+
if !ok {
54+
t.Errorf("missing expected field %q", field)
55+
continue
56+
}
57+
if !reflect.DeepEqual(actual, expected) {
58+
t.Errorf("field %q: expected %v (%T), got %v (%T)", field, expected, expected, actual, actual)
59+
}
60+
}
61+
}
62+
63+
func BenchmarkGreyNoiseAnnotator(b *testing.B) {
64+
factory := &GreyNoiseAnnotatorFactory{DBPath: "./data-snapshots/greynoise.mmdb"}
65+
err := factory.Initialize(nil)
66+
if err != nil {
67+
b.Fatalf("Error initializing greynoise annotator factory: %v", err)
68+
}
69+
a := factory.MakeAnnotator(0).(*GreyNoiseAnnotator)
70+
err = a.Initialize()
71+
if err != nil {
72+
b.Fatalf("Error initializing greynoise annotator: %v", err)
73+
}
74+
75+
// Pre-generate random IPs so generation is not part of the benchmark
76+
ips := make([]net.IP, 1000)
77+
for i := range ips {
78+
ips[i] = net.IPv4(
79+
byte(rand.IntN(256)),
80+
byte(rand.IntN(256)),
81+
byte(rand.IntN(256)),
82+
byte(rand.IntN(256)),
83+
)
84+
}
85+
86+
b.ResetTimer()
87+
b.ReportAllocs()
88+
for i := range b.N {
89+
a.Annotate(ips[i%len(ips)])
90+
}
91+
}

0 commit comments

Comments
 (0)