diff --git a/README.md b/README.md index 23a9f67..0739622 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,46 @@ Example Output: {"ip":"1.1.1.1","spur":{"as":{"number":13335,"organization":"Cloudflare, Inc."},"infrastructure":"DATACENTER","ip":"1.1.1.1","location":{"city":"Anycast","country":"ZZ","state":"Anycast"},"organization":"Taguchi Digital Marketing System"}} ``` +### Greynoise Psychic +[Greynoise](https://greynoise.io) is an IP intelligence feed that provides metadata like threat classification and associated CVE's. +Their Psychic data downloads provide their data feed in a database suitable for offline data enrichment. +To use their download with `zannotate`, you'll want to download an `.mmdb` formatted file using your GreyNoise API key. +As of April 2026, signing up with a free account gives access to data downloads. + +0. Sign up for a free GreyNoise account [here](https://www.greynoise.io). +1. Copy API key from the appropriate [section of your account](https://viz.greynoise.io/workspace/api-key). +2. Download a `mmdb` file. Details on download parameters +(The below command is for downloading data for a single date - April 7th, 2026 - you can also download data for a range of days and for models of various levels of detail. +See GreyNoise's Psychic [documentation](https://psychic.labs.greynoise.io) for more details. +```shell +curl -H "key: GREYNOISE_API_KEY_HERE" \ + https://psychic.labs.greynoise.io/v1/psychic/download/2026-04-07/3/mmdb \ + -o /tmp/m3.mmdb +``` + +3. Test GreyNoise data enrichment: + +> [!NOTE] +> The below examples are using the exact data download from the above `curl` command. What results you see will depend on the data downloaded. + +```shell +echo "14.1.105.157" | zannotate --greynoise --greynoise-database=/tmp/m3.mmdb +```` +Example Output: +```json +{"greynoise":{"classification":"malicious","cves":["CVE-2015-2051","CVE-2016-20016","CVE-2018-10561","CVE-2018-10562","CVE-2016-6277","CVE-2024-12847"],"date":"2026-04-07","handshake_complete":true,"last_seen":"2026-04-07T00:00:00Z","seen":true,"tags":["Mirai TCP Scanner","Mirai","Telnet Protocol","Generic IoT Default Password Attempt","Web Crawler","Generic Suspicious Linux Command in Request","HNAP Crawler","Telnet Login Attempt","D-Link Devices HNAP SOAPAction Header RCE Attempt","MVPower CCTV DVR RCE CVE-2016-20016 Attempt","JAWS Webserver RCE","GPON CVE-2018-10561 Router Worm","Generic ${IFS} Use in RCE Attempt","CCTV-DVR RCE","NETGEAR Command Injection CVE-2016-6277","NETGEAR DGN setup.cgi CVE-2024-12847 Command Execution Attempt","CGI Script Scanner"],"actor":"unknown"},"ip":"14.1.105.157"} +``` + +Note that many IPs will not be in the GreyNoise dataset, so you may see output like the following: +```shell +echo "1.1.1.1" | zannotate --greynoise --greynoise-database=/tmp/m3.mmdb +``` + +```json +{"ip":"1.1.1.1","greynoise":null} +``` + + # Input/Output ## Output diff --git a/data-snapshots/greynoise.mmdb b/data-snapshots/greynoise.mmdb new file mode 100644 index 0000000..5e42160 Binary files /dev/null and b/data-snapshots/greynoise.mmdb differ diff --git a/greynoise_psychic.go b/greynoise_psychic.go new file mode 100644 index 0000000..809526a --- /dev/null +++ b/greynoise_psychic.go @@ -0,0 +1,120 @@ +/* + * ZAnnotate Copyright 2026 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package zannotate + +import ( + "errors" + "flag" + "fmt" + "net" + "net/netip" + "os" + + "github.com/oschwald/maxminddb-golang/v2" + log "github.com/sirupsen/logrus" +) + +type GreyNoiseAnnotatorFactory struct { + BasePluginConf + DBPath string // path to the .mmdb path + greynoiseDB *maxminddb.Reader +} + +// GreyNoise Annotator Factory (Global) + +func (a *GreyNoiseAnnotatorFactory) MakeAnnotator(i int) Annotator { + var v GreyNoiseAnnotator + v.Factory = a + v.Id = i + return &v +} + +func (a *GreyNoiseAnnotatorFactory) Initialize(_ *GlobalConf) error { + if len(a.DBPath) == 0 { + return errors.New("greynoise database path is required when greynoise annotator is enabled, use --greynoise-database") + } + data, err := os.ReadFile(a.DBPath) // ensure DB is read in-memory + if err != nil { + return fmt.Errorf("unable to read greynoise database at %s: %w", a.DBPath, err) + } + a.greynoiseDB, err = maxminddb.OpenBytes(data) + if err != nil { + return fmt.Errorf("unable to open greynoise database at %s: %w", a.DBPath, err) + } + return nil +} + +func (a *GreyNoiseAnnotatorFactory) GetWorkers() int { + return a.Threads +} + +func (a *GreyNoiseAnnotatorFactory) Close() error { + if err := a.greynoiseDB.Close(); err != nil { + return fmt.Errorf("unable to close greynoise database at %s: %w", a.DBPath, err) + } + return nil +} + +func (a *GreyNoiseAnnotatorFactory) IsEnabled() bool { + return a.Enabled +} + +func (a *GreyNoiseAnnotatorFactory) AddFlags(flags *flag.FlagSet) { + // Reverse DNS Lookup + flags.BoolVar(&a.Enabled, "greynoise", false, "greynoise psychic data intelligence") + flags.StringVar(&a.DBPath, "greynoise-database", "", "path to greynoise psychic .mmdb file") + flags.IntVar(&a.Threads, "greynoise-threads", 2, "how many enrichment threads to use") +} + +// GreyNoiseAnnotator (Per-Worker) +type GreyNoiseAnnotator struct { + Factory *GreyNoiseAnnotatorFactory + Id int +} + +func (a *GreyNoiseAnnotator) Initialize() (err error) { + return nil +} + +func (a *GreyNoiseAnnotator) GetFieldName() string { + return "greynoise" +} + +// Annotate performs a reverse DNS lookup for the given IP address and returns the results. +// If an error occurs or a lookup fails, it returns nil +func (a *GreyNoiseAnnotator) Annotate(ip net.IP) interface{} { + addr, ok := netip.AddrFromSlice(ip) + if !ok { + log.Debugf("unable to convert IP %s to address", ip) + return nil + } + addr = addr.Unmap() + var result any + err := a.Factory.greynoiseDB.Lookup(addr).Decode(&result) + if err != nil { + log.Debugf("unable to annotate IP (%s): %v", addr, err) + return nil + } + return result +} + +func (a *GreyNoiseAnnotator) Close() error { + return nil +} + +func init() { + s := new(GreyNoiseAnnotatorFactory) + RegisterAnnotator(s) +} diff --git a/greynoise_psychic_test.go b/greynoise_psychic_test.go new file mode 100644 index 0000000..d4467b1 --- /dev/null +++ b/greynoise_psychic_test.go @@ -0,0 +1,91 @@ +/* + * ZAnnotate Copyright 2025 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package zannotate + +import ( + "math/rand/v2" + "net" + "reflect" + "testing" +) + +// TestGreyNoiseAnnotator tests that given a database file and a known IP, the annotator returns the expected values for that IP in the DB +func TestGreyNoiseAnnotator(t *testing.T) { + expectedFields := map[string]any{ + "actor": "Stanford University", + "classification": "benign", + "date": "2026-04-07", + "handshake_complete": true, + "last_seen": "2026-04-07T00:00:00Z", + "seen": true, + "tags": []any{"Stanford University", "RDP Crawler", "RDP Protocol"}, + } + factory := &GreyNoiseAnnotatorFactory{DBPath: "./data-snapshots/greynoise.mmdb"} + err := factory.Initialize(nil) + if err != nil { + t.Fatalf("Error initializing greynoise annotator factory: %v", err) + } + a := factory.MakeAnnotator(0).(*GreyNoiseAnnotator) + err = a.Initialize() + if err != nil { + t.Fatalf("Error initializing greynoise annotator: %v", err) + } + + ip := "171.67.71.209" + res := a.Annotate(net.ParseIP(ip)) + if res == nil { + t.Fatalf("GreyNoiseAnnotator failed to annotate %s", ip) + } + + for field, expected := range expectedFields { + actual, ok := res.(map[string]any)[field] + if !ok { + t.Errorf("missing expected field %q", field) + continue + } + if !reflect.DeepEqual(actual, expected) { + t.Errorf("field %q: expected %v (%T), got %v (%T)", field, expected, expected, actual, actual) + } + } +} + +func BenchmarkGreyNoiseAnnotator(b *testing.B) { + factory := &GreyNoiseAnnotatorFactory{DBPath: "./data-snapshots/greynoise.mmdb"} + err := factory.Initialize(nil) + if err != nil { + b.Fatalf("Error initializing greynoise annotator factory: %v", err) + } + a := factory.MakeAnnotator(0).(*GreyNoiseAnnotator) + err = a.Initialize() + if err != nil { + b.Fatalf("Error initializing greynoise annotator: %v", err) + } + + // Pre-generate random IPs so generation is not part of the benchmark + ips := make([]net.IP, 1000) + for i := range ips { + ips[i] = net.IPv4( + byte(rand.IntN(256)), + byte(rand.IntN(256)), + byte(rand.IntN(256)), + byte(rand.IntN(256)), + ) + } + + b.ResetTimer() + b.ReportAllocs() + for i := range b.N { + a.Annotate(ips[i%len(ips)]) + } +}