diff --git a/aop/traffic/exchange.go b/aop/traffic/exchange.go new file mode 100644 index 00000000..f625ad03 --- /dev/null +++ b/aop/traffic/exchange.go @@ -0,0 +1,264 @@ +package traffic + +import ( + "encoding/json" + "sort" +) + +// Pair is one HTTP header line: flat, ordered, duplicates preserved. It is the +// canonical header form both on the wire (proto Header) and in memory; a map +// cannot express order or repeated names. +type Pair struct { + Name string + Value string +} + +// Request is the request half of an exchange. +type Request struct { + Method string + URL string + Protocol string + Headers []Pair + Body []byte +} + +// Response is the response half of an exchange. It is optional on Exchange: a +// request that never got a response (timeout, refused connection, one-way +// capture) has no response half. +type Response struct { + StatusCode int + ReasonPhrase string + Headers []Pair + Body []byte +} + +// Exchange is the canonical in-memory form of one captured HTTP exchange, +// composed of a request and an optional response. The Flow proto message is +// its wire view; the two are one model, converted by ExchangeFromFlow and +// Proto. +// +// Its JSON form is the flow element of the http.exchange.v1 evidence payload, +// where headers serialize as a name→values map for compatibility with the +// stored contract. Order and duplicate names survive in memory; the map view +// is the persisted projection. +type Exchange struct { + ID string + Request Request + Response *Response + Error string + Complete bool +} + +// exchangeJSON is the persisted shape: identical field names and order to the +// http.exchange.v1 flow element, headers as a name→values map. +type exchangeJSON struct { + ID string `json:"id"` + Request requestJSON `json:"request"` + Response *responseJSON `json:"response,omitempty"` + Error string `json:"error,omitempty"` + Complete bool `json:"complete"` +} + +type requestJSON struct { + Method string `json:"method"` + URL string `json:"url"` + Protocol string `json:"protocol,omitempty"` + Headers map[string][]string `json:"headers,omitempty"` + Body []byte `json:"body,omitempty"` +} + +type responseJSON struct { + StatusCode int `json:"status_code"` + ReasonPhrase string `json:"reason_phrase,omitempty"` + Headers map[string][]string `json:"headers,omitempty"` + Body []byte `json:"body,omitempty"` +} + +func (e Exchange) MarshalJSON() ([]byte, error) { + wire := exchangeJSON{ + ID: e.ID, + Request: requestJSON{ + Method: e.Request.Method, + URL: e.Request.URL, + Protocol: e.Request.Protocol, + Headers: pairsToMap(e.Request.Headers), + Body: e.Request.Body, + }, + Error: e.Error, + Complete: e.Complete, + } + if e.Response != nil { + wire.Response = &responseJSON{ + StatusCode: e.Response.StatusCode, + ReasonPhrase: e.Response.ReasonPhrase, + Headers: pairsToMap(e.Response.Headers), + Body: e.Response.Body, + } + } + return json.Marshal(wire) +} + +func (e *Exchange) UnmarshalJSON(data []byte) error { + var wire exchangeJSON + if err := json.Unmarshal(data, &wire); err != nil { + return err + } + *e = Exchange{ + ID: wire.ID, + Request: Request{ + Method: wire.Request.Method, + URL: wire.Request.URL, + Protocol: wire.Request.Protocol, + Headers: mapToPairs(wire.Request.Headers), + Body: wire.Request.Body, + }, + Error: wire.Error, + Complete: wire.Complete, + } + if wire.Response != nil { + e.Response = &Response{ + StatusCode: wire.Response.StatusCode, + ReasonPhrase: wire.Response.ReasonPhrase, + Headers: mapToPairs(wire.Response.Headers), + Body: wire.Response.Body, + } + } + return nil +} + +// pairsToMap folds a pair sequence into the persisted map view, merging +// duplicate names in encounter order. Nil when empty so the key is omitted. +func pairsToMap(pairs []Pair) map[string][]string { + if len(pairs) == 0 { + return nil + } + out := make(map[string][]string, len(pairs)) + for _, p := range pairs { + out[p.Name] = append(out[p.Name], p.Value) + } + return out +} + +// mapToPairs unfolds the persisted map view. Keys are sorted so the in-memory +// form is deterministic even though the map lost the original order. +func mapToPairs(headers map[string][]string) []Pair { + if len(headers) == 0 { + return nil + } + names := make([]string, 0, len(headers)) + for name := range headers { + names = append(names, name) + } + sort.Strings(names) + out := make([]Pair, 0, len(headers)) + for _, name := range names { + for _, value := range headers[name] { + out = append(out, Pair{Name: name, Value: value}) + } + } + return out +} + +// ExchangeFromFlow lifts a wire Flow into its canonical form. ToolId and +// Timestamp are attribution and transport metadata, not exchange semantics, so +// they do not cross over. +func ExchangeFromFlow(f *Flow) *Exchange { + if f == nil { + return nil + } + e := &Exchange{ + ID: f.GetId(), + Request: requestFromProto(f.GetRequest()), + Error: f.GetError(), + Complete: f.GetComplete(), + } + if r := f.GetResponse(); r != nil { + resp := responseFromProto(r) + e.Response = &resp + } + return e +} + +// Proto renders the exchange as a wire Flow. Attribution (ToolId, Timestamp) +// is the caller's to stamp. +func (e *Exchange) Proto() *Flow { + if e == nil { + return nil + } + f := &Flow{ + Id: e.ID, + Request: requestToProto(e.Request), + Error: e.Error, + Complete: e.Complete, + } + if e.Response != nil { + f.Response = responseToProto(*e.Response) + } + return f +} + +func requestFromProto(r *HttpRequest) Request { + if r == nil { + return Request{} + } + return Request{ + Method: r.GetMethod(), + URL: r.GetUrl(), + Protocol: r.GetProtocol(), + Headers: pairsFromProto(r.GetHeaders()), + Body: r.GetBody(), + } +} + +func responseFromProto(r *HttpResponse) Response { + return Response{ + StatusCode: int(r.GetStatusCode()), + ReasonPhrase: r.GetReasonPhrase(), + Headers: pairsFromProto(r.GetHeaders()), + Body: r.GetBody(), + } +} + +func requestToProto(r Request) *HttpRequest { + return &HttpRequest{ + Method: r.Method, + Url: r.URL, + Protocol: r.Protocol, + Headers: pairsToProto(r.Headers), + Body: r.Body, + } +} + +func responseToProto(r Response) *HttpResponse { + return &HttpResponse{ + StatusCode: int32(r.StatusCode), + ReasonPhrase: r.ReasonPhrase, + Headers: pairsToProto(r.Headers), + Body: r.Body, + } +} + +func pairsFromProto(headers []*Header) []Pair { + if len(headers) == 0 { + return nil + } + out := make([]Pair, 0, len(headers)) + for _, h := range headers { + if h == nil { + continue + } + out = append(out, Pair{Name: h.GetName(), Value: h.GetValue()}) + } + return out +} + +func pairsToProto(pairs []Pair) []*Header { + if len(pairs) == 0 { + return nil + } + out := make([]*Header, 0, len(pairs)) + for _, p := range pairs { + out = append(out, &Header{Name: p.Name, Value: p.Value}) + } + return out +} diff --git a/aop/traffic/exchange_test.go b/aop/traffic/exchange_test.go new file mode 100644 index 00000000..978078ce --- /dev/null +++ b/aop/traffic/exchange_test.go @@ -0,0 +1,142 @@ +package traffic + +import ( + "encoding/json" + "testing" +) + +func TestFlowExchangeRoundTrip(t *testing.T) { + flow := &Flow{ + Id: "flow-1", + ToolId: "call-9", + Request: &HttpRequest{ + Method: "POST", + Url: "https://example.test/login", + Protocol: "HTTP/2.0", + Headers: []*Header{ + {Name: "X-Trace", Value: "a"}, + {Name: "X-Trace", Value: "b"}, + {Name: "Content-Type", Value: "application/json"}, + }, + Body: []byte(`{"u":"n"}`), + }, + Response: &HttpResponse{ + StatusCode: 302, + ReasonPhrase: "Found", + Headers: []*Header{{Name: "Location", Value: "/home"}}, + }, + Complete: true, + } + + exchange := ExchangeFromFlow(flow) + if exchange.ID != "flow-1" || exchange.Response == nil || exchange.Response.StatusCode != 302 || !exchange.Complete { + t.Fatalf("scalar fields did not cross: %#v", exchange) + } + if len(exchange.Request.Headers) != 3 || exchange.Request.Headers[1] != (Pair{Name: "X-Trace", Value: "b"}) { + t.Fatalf("duplicate headers lost order or values: %#v", exchange.Request.Headers) + } + + back := exchange.Proto() + if back.GetToolId() != "" { + t.Fatal("attribution must not cross into the exchange model") + } + if back.GetId() != flow.GetId() || back.GetResponse().GetReasonPhrase() != "Found" || len(back.GetRequest().GetHeaders()) != 3 { + t.Fatalf("proto round-trip mismatch: %#v", back) + } +} + +func TestExchangeRequestOnly(t *testing.T) { + flow := &Flow{ + Id: "flow-2", + Request: &HttpRequest{Method: "GET", Url: "http://unreachable.test/"}, + Error: "dial tcp: connection refused", + } + exchange := ExchangeFromFlow(flow) + if exchange.Response != nil { + t.Fatalf("request-only flow gained a response: %#v", exchange.Response) + } + if exchange.Complete { + t.Fatal("request-only flow must not be complete") + } + back := exchange.Proto() + if back.GetResponse() != nil { + t.Fatal("response must stay absent on the wire") + } +} + +func TestExchangeNilSafety(t *testing.T) { + if ExchangeFromFlow(nil) != nil { + t.Fatal("nil flow produced a non-nil exchange") + } + var exchange *Exchange + if exchange.Proto() != nil { + t.Fatal("nil exchange produced a non-nil flow") + } +} + +// TestExchangeJSONMatchesV1EvidenceShape pins the persisted form: the flow +// element of an http.exchange.v1 payload, headers as a name→values map. +func TestExchangeJSONMatchesV1EvidenceShape(t *testing.T) { + const v1 = `{"id":"flow-1","request":{"method":"GET","url":"https://example.test/",` + + `"headers":{"Accept":["text/html"],"X-Trace-Id":["a","b"]}},` + + `"response":{"status_code":200,"body":"aGVsbG8="},"complete":true}` + + var exchange Exchange + if err := json.Unmarshal([]byte(v1), &exchange); err != nil { + t.Fatalf("decode v1 flow: %v", err) + } + if len(exchange.Request.Headers) != 3 { + t.Fatalf("headers did not unfold to pairs: %#v", exchange.Request.Headers) + } + if exchange.Response == nil || exchange.Response.StatusCode != 200 { + t.Fatalf("response did not cross: %#v", exchange.Response) + } + + data, err := json.Marshal(exchange) + if err != nil { + t.Fatal(err) + } + if string(data) != v1 { + t.Fatalf("persisted shape drifted:\n got %s\nwant %s", data, v1) + } +} + +// TestExchangeJSONRequestOnly pins the persisted form of an exchange that never +// got a response: no response key at all. +func TestExchangeJSONRequestOnly(t *testing.T) { + data, err := json.Marshal(Exchange{ + ID: "f", + Request: Request{Method: "GET", URL: "http://x/"}, + Error: "dial tcp: timeout", + }) + if err != nil { + t.Fatal(err) + } + const want = `{"id":"f","request":{"method":"GET","url":"http://x/"},"error":"dial tcp: timeout","complete":false}` + if string(data) != want { + t.Fatalf(" got %s\nwant %s", data, want) + } + + var exchange Exchange + if err := json.Unmarshal([]byte(want), &exchange); err != nil { + t.Fatal(err) + } + if exchange.Response != nil { + t.Fatal("absent response key must stay nil") + } +} + +func TestExchangeJSONOmitsEmptyFields(t *testing.T) { + data, err := json.Marshal(Exchange{ + ID: "f", + Request: Request{Method: "GET", URL: "http://x/"}, + Response: &Response{StatusCode: 200}, + }) + if err != nil { + t.Fatal(err) + } + const want = `{"id":"f","request":{"method":"GET","url":"http://x/"},"response":{"status_code":200},"complete":false}` + if string(data) != want { + t.Fatalf(" got %s\nwant %s", data, want) + } +} diff --git a/aop/traffic/protocol.pb.go b/aop/traffic/protocol.pb.go index ea812ec2..eb057487 100644 --- a/aop/traffic/protocol.pb.go +++ b/aop/traffic/protocol.pb.go @@ -709,43 +709,32 @@ func (x *Header) GetValue() string { return "" } -// Flow is one captured request/response pair. Its fields mirror the consumer's -// http.exchange shape so a consumer can map it directly; tool_id is the AOP -// tool-call id whose egress produced this flow. -type Flow struct { - state protoimpl.MessageState `protogen:"open.v1"` - Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` - ToolId string `protobuf:"bytes,2,opt,name=tool_id,json=toolId,proto3" json:"tool_id,omitempty"` - Method string `protobuf:"bytes,3,opt,name=method,proto3" json:"method,omitempty"` - Url string `protobuf:"bytes,4,opt,name=url,proto3" json:"url,omitempty"` - Protocol string `protobuf:"bytes,5,opt,name=protocol,proto3" json:"protocol,omitempty"` - StatusCode int32 `protobuf:"varint,6,opt,name=status_code,json=statusCode,proto3" json:"status_code,omitempty"` - ReasonPhrase string `protobuf:"bytes,7,opt,name=reason_phrase,json=reasonPhrase,proto3" json:"reason_phrase,omitempty"` - RequestHeaders []*Header `protobuf:"bytes,8,rep,name=request_headers,json=requestHeaders,proto3" json:"request_headers,omitempty"` - ResponseHeaders []*Header `protobuf:"bytes,9,rep,name=response_headers,json=responseHeaders,proto3" json:"response_headers,omitempty"` - RequestBody []byte `protobuf:"bytes,10,opt,name=request_body,json=requestBody,proto3" json:"request_body,omitempty"` - ResponseBody []byte `protobuf:"bytes,11,opt,name=response_body,json=responseBody,proto3" json:"response_body,omitempty"` - Error string `protobuf:"bytes,12,opt,name=error,proto3" json:"error,omitempty"` - Complete bool `protobuf:"varint,13,opt,name=complete,proto3" json:"complete,omitempty"` - Timestamp *timestamppb.Timestamp `protobuf:"bytes,14,opt,name=timestamp,proto3" json:"timestamp,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache +// HttpRequest is the request half of an exchange. +type HttpRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Method string `protobuf:"bytes,1,opt,name=method,proto3" json:"method,omitempty"` + Url string `protobuf:"bytes,2,opt,name=url,proto3" json:"url,omitempty"` + Protocol string `protobuf:"bytes,3,opt,name=protocol,proto3" json:"protocol,omitempty"` + Headers []*Header `protobuf:"bytes,4,rep,name=headers,proto3" json:"headers,omitempty"` + Body []byte `protobuf:"bytes,5,opt,name=body,proto3" json:"body,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *Flow) Reset() { - *x = Flow{} +func (x *HttpRequest) Reset() { + *x = HttpRequest{} mi := &file_aop_traffic_protocol_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *Flow) String() string { +func (x *HttpRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*Flow) ProtoMessage() {} +func (*HttpRequest) ProtoMessage() {} -func (x *Flow) ProtoReflect() protoreflect.Message { +func (x *HttpRequest) ProtoReflect() protoreflect.Message { mi := &file_aop_traffic_protocol_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -757,86 +746,176 @@ func (x *Flow) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use Flow.ProtoReflect.Descriptor instead. -func (*Flow) Descriptor() ([]byte, []int) { +// Deprecated: Use HttpRequest.ProtoReflect.Descriptor instead. +func (*HttpRequest) Descriptor() ([]byte, []int) { return file_aop_traffic_protocol_proto_rawDescGZIP(), []int{9} } -func (x *Flow) GetId() string { +func (x *HttpRequest) GetMethod() string { if x != nil { - return x.Id + return x.Method } return "" } -func (x *Flow) GetToolId() string { +func (x *HttpRequest) GetUrl() string { if x != nil { - return x.ToolId + return x.Url } return "" } -func (x *Flow) GetMethod() string { +func (x *HttpRequest) GetProtocol() string { if x != nil { - return x.Method + return x.Protocol } return "" } -func (x *Flow) GetUrl() string { +func (x *HttpRequest) GetHeaders() []*Header { if x != nil { - return x.Url + return x.Headers } - return "" + return nil } -func (x *Flow) GetProtocol() string { +func (x *HttpRequest) GetBody() []byte { if x != nil { - return x.Protocol + return x.Body } - return "" + return nil } -func (x *Flow) GetStatusCode() int32 { +// HttpResponse is the response half of an exchange. It is optional on Flow: a +// request that never got a response (timeout, refused connection, one-way +// capture) has no response half. +type HttpResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + StatusCode int32 `protobuf:"varint,1,opt,name=status_code,json=statusCode,proto3" json:"status_code,omitempty"` + ReasonPhrase string `protobuf:"bytes,2,opt,name=reason_phrase,json=reasonPhrase,proto3" json:"reason_phrase,omitempty"` + Headers []*Header `protobuf:"bytes,3,rep,name=headers,proto3" json:"headers,omitempty"` + Body []byte `protobuf:"bytes,4,opt,name=body,proto3" json:"body,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HttpResponse) Reset() { + *x = HttpResponse{} + mi := &file_aop_traffic_protocol_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HttpResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HttpResponse) ProtoMessage() {} + +func (x *HttpResponse) ProtoReflect() protoreflect.Message { + mi := &file_aop_traffic_protocol_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HttpResponse.ProtoReflect.Descriptor instead. +func (*HttpResponse) Descriptor() ([]byte, []int) { + return file_aop_traffic_protocol_proto_rawDescGZIP(), []int{10} +} + +func (x *HttpResponse) GetStatusCode() int32 { if x != nil { return x.StatusCode } return 0 } -func (x *Flow) GetReasonPhrase() string { +func (x *HttpResponse) GetReasonPhrase() string { if x != nil { return x.ReasonPhrase } return "" } -func (x *Flow) GetRequestHeaders() []*Header { +func (x *HttpResponse) GetHeaders() []*Header { if x != nil { - return x.RequestHeaders + return x.Headers } return nil } -func (x *Flow) GetResponseHeaders() []*Header { +func (x *HttpResponse) GetBody() []byte { if x != nil { - return x.ResponseHeaders + return x.Body } return nil } -func (x *Flow) GetRequestBody() []byte { +// Flow is one captured request/response exchange. Its nested shape mirrors the +// consumer's http.exchange form so a consumer can map it directly; tool_id is +// the AOP tool-call id whose egress produced this flow. Fields 3-11 were the +// pre-nesting flat shape. +type Flow struct { + state protoimpl.MessageState `protogen:"open.v1"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + ToolId string `protobuf:"bytes,2,opt,name=tool_id,json=toolId,proto3" json:"tool_id,omitempty"` + Error string `protobuf:"bytes,12,opt,name=error,proto3" json:"error,omitempty"` + Complete bool `protobuf:"varint,13,opt,name=complete,proto3" json:"complete,omitempty"` + Timestamp *timestamppb.Timestamp `protobuf:"bytes,14,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + Request *HttpRequest `protobuf:"bytes,15,opt,name=request,proto3" json:"request,omitempty"` + Response *HttpResponse `protobuf:"bytes,16,opt,name=response,proto3" json:"response,omitempty"` // absent when no response was received + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Flow) Reset() { + *x = Flow{} + mi := &file_aop_traffic_protocol_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Flow) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Flow) ProtoMessage() {} + +func (x *Flow) ProtoReflect() protoreflect.Message { + mi := &file_aop_traffic_protocol_proto_msgTypes[11] if x != nil { - return x.RequestBody + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms } - return nil + return mi.MessageOf(x) +} + +// Deprecated: Use Flow.ProtoReflect.Descriptor instead. +func (*Flow) Descriptor() ([]byte, []int) { + return file_aop_traffic_protocol_proto_rawDescGZIP(), []int{11} } -func (x *Flow) GetResponseBody() []byte { +func (x *Flow) GetId() string { if x != nil { - return x.ResponseBody + return x.Id } - return nil + return "" +} + +func (x *Flow) GetToolId() string { + if x != nil { + return x.ToolId + } + return "" } func (x *Flow) GetError() string { @@ -860,6 +939,20 @@ func (x *Flow) GetTimestamp() *timestamppb.Timestamp { return nil } +func (x *Flow) GetRequest() *HttpRequest { + if x != nil { + return x.Request + } + return nil +} + +func (x *Flow) GetResponse() *HttpResponse { + if x != nil { + return x.Response + } + return nil +} + type ProtocolMessage struct { state protoimpl.MessageState `protogen:"open.v1"` // Types that are valid to be assigned to Message: @@ -875,7 +968,7 @@ type ProtocolMessage struct { func (x *ProtocolMessage) Reset() { *x = ProtocolMessage{} - mi := &file_aop_traffic_protocol_proto_msgTypes[10] + mi := &file_aop_traffic_protocol_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -887,7 +980,7 @@ func (x *ProtocolMessage) String() string { func (*ProtocolMessage) ProtoMessage() {} func (x *ProtocolMessage) ProtoReflect() protoreflect.Message { - mi := &file_aop_traffic_protocol_proto_msgTypes[10] + mi := &file_aop_traffic_protocol_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -900,7 +993,7 @@ func (x *ProtocolMessage) ProtoReflect() protoreflect.Message { // Deprecated: Use ProtocolMessage.ProtoReflect.Descriptor instead. func (*ProtocolMessage) Descriptor() ([]byte, []int) { - return file_aop_traffic_protocol_proto_rawDescGZIP(), []int{10} + return file_aop_traffic_protocol_proto_rawDescGZIP(), []int{12} } func (x *ProtocolMessage) GetMessage() isProtocolMessage_Message { @@ -1020,24 +1113,27 @@ const file_aop_traffic_protocol_proto_rawDesc = "" + "\x05error\x18\x03 \x01(\tR\x05error\"2\n" + "\x06Header\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value\"\xed\x03\n" + + "\x05value\x18\x02 \x01(\tR\x05value\"\x96\x01\n" + + "\vHttpRequest\x12\x16\n" + + "\x06method\x18\x01 \x01(\tR\x06method\x12\x10\n" + + "\x03url\x18\x02 \x01(\tR\x03url\x12\x1a\n" + + "\bprotocol\x18\x03 \x01(\tR\bprotocol\x12-\n" + + "\aheaders\x18\x04 \x03(\v2\x13.aop.traffic.HeaderR\aheaders\x12\x12\n" + + "\x04body\x18\x05 \x01(\fR\x04body\"\x97\x01\n" + + "\fHttpResponse\x12\x1f\n" + + "\vstatus_code\x18\x01 \x01(\x05R\n" + + "statusCode\x12#\n" + + "\rreason_phrase\x18\x02 \x01(\tR\freasonPhrase\x12-\n" + + "\aheaders\x18\x03 \x03(\v2\x13.aop.traffic.HeaderR\aheaders\x12\x12\n" + + "\x04body\x18\x04 \x01(\fR\x04body\"\x8c\x02\n" + "\x04Flow\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x12\x17\n" + - "\atool_id\x18\x02 \x01(\tR\x06toolId\x12\x16\n" + - "\x06method\x18\x03 \x01(\tR\x06method\x12\x10\n" + - "\x03url\x18\x04 \x01(\tR\x03url\x12\x1a\n" + - "\bprotocol\x18\x05 \x01(\tR\bprotocol\x12\x1f\n" + - "\vstatus_code\x18\x06 \x01(\x05R\n" + - "statusCode\x12#\n" + - "\rreason_phrase\x18\a \x01(\tR\freasonPhrase\x12<\n" + - "\x0frequest_headers\x18\b \x03(\v2\x13.aop.traffic.HeaderR\x0erequestHeaders\x12>\n" + - "\x10response_headers\x18\t \x03(\v2\x13.aop.traffic.HeaderR\x0fresponseHeaders\x12!\n" + - "\frequest_body\x18\n" + - " \x01(\fR\vrequestBody\x12#\n" + - "\rresponse_body\x18\v \x01(\fR\fresponseBody\x12\x14\n" + + "\atool_id\x18\x02 \x01(\tR\x06toolId\x12\x14\n" + "\x05error\x18\f \x01(\tR\x05error\x12\x1a\n" + "\bcomplete\x18\r \x01(\bR\bcomplete\x128\n" + - "\ttimestamp\x18\x0e \x01(\v2\x1a.google.protobuf.TimestampR\ttimestamp\"\xd5\x01\n" + + "\ttimestamp\x18\x0e \x01(\v2\x1a.google.protobuf.TimestampR\ttimestamp\x122\n" + + "\arequest\x18\x0f \x01(\v2\x18.aop.traffic.HttpRequestR\arequest\x125\n" + + "\bresponse\x18\x10 \x01(\v2\x19.aop.traffic.HttpResponseR\bresponseJ\x04\b\x03\x10\f\"\xd5\x01\n" + "\x0fProtocolMessage\x126\n" + "\tconfigure\x18\n" + " \x01(\v2\x16.aop.traffic.ConfigureH\x00R\tconfigure\x12*\n" + @@ -1071,7 +1167,7 @@ func file_aop_traffic_protocol_proto_rawDescGZIP() []byte { } var file_aop_traffic_protocol_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_aop_traffic_protocol_proto_msgTypes = make([]protoimpl.MessageInfo, 11) +var file_aop_traffic_protocol_proto_msgTypes = make([]protoimpl.MessageInfo, 13) var file_aop_traffic_protocol_proto_goTypes = []any{ (CaptureMode)(0), // 0: aop.traffic.CaptureMode (RoutingMode)(0), // 1: aop.traffic.RoutingMode @@ -1084,9 +1180,11 @@ var file_aop_traffic_protocol_proto_goTypes = []any{ (*CaptureState)(nil), // 8: aop.traffic.CaptureState (*State)(nil), // 9: aop.traffic.State (*Header)(nil), // 10: aop.traffic.Header - (*Flow)(nil), // 11: aop.traffic.Flow - (*ProtocolMessage)(nil), // 12: aop.traffic.ProtocolMessage - (*timestamppb.Timestamp)(nil), // 13: google.protobuf.Timestamp + (*HttpRequest)(nil), // 11: aop.traffic.HttpRequest + (*HttpResponse)(nil), // 12: aop.traffic.HttpResponse + (*Flow)(nil), // 13: aop.traffic.Flow + (*ProtocolMessage)(nil), // 14: aop.traffic.ProtocolMessage + (*timestamppb.Timestamp)(nil), // 15: google.protobuf.Timestamp } var file_aop_traffic_protocol_proto_depIdxs = []int32{ 1, // 0: aop.traffic.RoutingConfig.mode:type_name -> aop.traffic.RoutingMode @@ -1098,18 +1196,20 @@ var file_aop_traffic_protocol_proto_depIdxs = []int32{ 0, // 6: aop.traffic.CaptureState.mode:type_name -> aop.traffic.CaptureMode 7, // 7: aop.traffic.State.routing:type_name -> aop.traffic.RoutingState 8, // 8: aop.traffic.State.capture:type_name -> aop.traffic.CaptureState - 10, // 9: aop.traffic.Flow.request_headers:type_name -> aop.traffic.Header - 10, // 10: aop.traffic.Flow.response_headers:type_name -> aop.traffic.Header - 13, // 11: aop.traffic.Flow.timestamp:type_name -> google.protobuf.Timestamp - 5, // 12: aop.traffic.ProtocolMessage.configure:type_name -> aop.traffic.Configure - 6, // 13: aop.traffic.ProtocolMessage.query:type_name -> aop.traffic.Query - 9, // 14: aop.traffic.ProtocolMessage.state:type_name -> aop.traffic.State - 11, // 15: aop.traffic.ProtocolMessage.flow:type_name -> aop.traffic.Flow - 16, // [16:16] is the sub-list for method output_type - 16, // [16:16] is the sub-list for method input_type - 16, // [16:16] is the sub-list for extension type_name - 16, // [16:16] is the sub-list for extension extendee - 0, // [0:16] is the sub-list for field type_name + 10, // 9: aop.traffic.HttpRequest.headers:type_name -> aop.traffic.Header + 10, // 10: aop.traffic.HttpResponse.headers:type_name -> aop.traffic.Header + 15, // 11: aop.traffic.Flow.timestamp:type_name -> google.protobuf.Timestamp + 11, // 12: aop.traffic.Flow.request:type_name -> aop.traffic.HttpRequest + 12, // 13: aop.traffic.Flow.response:type_name -> aop.traffic.HttpResponse + 5, // 14: aop.traffic.ProtocolMessage.configure:type_name -> aop.traffic.Configure + 6, // 15: aop.traffic.ProtocolMessage.query:type_name -> aop.traffic.Query + 9, // 16: aop.traffic.ProtocolMessage.state:type_name -> aop.traffic.State + 13, // 17: aop.traffic.ProtocolMessage.flow:type_name -> aop.traffic.Flow + 18, // [18:18] is the sub-list for method output_type + 18, // [18:18] is the sub-list for method input_type + 18, // [18:18] is the sub-list for extension type_name + 18, // [18:18] is the sub-list for extension extendee + 0, // [0:18] is the sub-list for field type_name } func init() { file_aop_traffic_protocol_proto_init() } @@ -1117,7 +1217,7 @@ func file_aop_traffic_protocol_proto_init() { if File_aop_traffic_protocol_proto != nil { return } - file_aop_traffic_protocol_proto_msgTypes[10].OneofWrappers = []any{ + file_aop_traffic_protocol_proto_msgTypes[12].OneofWrappers = []any{ (*ProtocolMessage_Configure)(nil), (*ProtocolMessage_Query)(nil), (*ProtocolMessage_State)(nil), @@ -1129,7 +1229,7 @@ func file_aop_traffic_protocol_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_aop_traffic_protocol_proto_rawDesc), len(file_aop_traffic_protocol_proto_rawDesc)), NumEnums: 2, - NumMessages: 11, + NumMessages: 13, NumExtensions: 0, NumServices: 0, }, diff --git a/cmd/aiscan/imports.go b/cmd/aiscan/imports.go index 73cd56db..4a85fb2e 100644 --- a/cmd/aiscan/imports.go +++ b/cmd/aiscan/imports.go @@ -6,6 +6,7 @@ package main import ( _ "github.com/chainreactors/aiscan/tools" _ "github.com/chainreactors/aiscan/tools/arsenal" + _ "github.com/chainreactors/aiscan/tools/curl" _ "github.com/chainreactors/aiscan/tools/gogo" _ "github.com/chainreactors/aiscan/tools/ioa" _ "github.com/chainreactors/aiscan/tools/neutron" diff --git a/cmd/runner/main.go b/cmd/runner/main.go index 948fed10..447bed13 100644 --- a/cmd/runner/main.go +++ b/cmd/runner/main.go @@ -18,6 +18,7 @@ import ( "github.com/chainreactors/aiscan/pkg/runner" _ "github.com/chainreactors/aiscan/tools" _ "github.com/chainreactors/aiscan/tools/arsenal" + _ "github.com/chainreactors/aiscan/tools/curl" _ "github.com/chainreactors/aiscan/tools/gogo" _ "github.com/chainreactors/aiscan/tools/ioa" _ "github.com/chainreactors/aiscan/tools/neutron" diff --git a/skills/aiscan/okf/easm/curl.md b/skills/aiscan/okf/easm/curl.md new file mode 100644 index 00000000..a8c7272c --- /dev/null +++ b/skills/aiscan/okf/easm/curl.md @@ -0,0 +1,50 @@ +--- +type: Tool Playbook +title: curl +description: Use this playbook for targeted HTTP requests with curl — a pure-Go, browser-naturalized, evidence-first client that replaces ad-hoc HTTP probing. +tags: [easm, web] +status: stable +--- + +# curl + +curl is aiscan's pure-Go HTTP client. It exposes a curl-shaped flag surface, so +it is used exactly like the system tool, while every request routes through the +runner proxy — attributed by tool-call id and captured as HTTP evidence — and +carries a browser-shaped User-Agent and header set by default instead of +announcing itself as automated tooling. + +Capabilities: + +- send one HTTP request with an explicit method, headers, and body +- submit form data (`-d`) or fold it into the query string (`-G`) +- follow redirects (`-L`), with a bounded redirect count (`--max-redirs`) +- carry and persist cookies across calls (`-b` / `-c`) +- override the naturalized User-Agent and headers when a specific client shape is needed +- include response headers (`-i`), write the body to a file (`-o`), and report + outcome fields (`-w`, e.g. `%{http_code}`, `%{url_effective}`) + +Common usage: + +```bash +curl +curl -X POST -d 'a=1&b=2' +curl -H 'Authorization: Bearer ...' -i +curl -L -b 'sid=abc' -c jar.txt +``` + +Notes: + +- Requests are recorded as HTTP evidence through the runner proxy; this is the + first-class path for evidence-backed HTTP probing. +- A browser User-Agent and header set are applied only where you did not set them; + `-A` and `-H` always win. +- Unsupported flags are rejected rather than silently ignored, so behavior is + never quietly different from what was asked. + +## Related concepts + +- Use [spray](spray.md) for breadth (many URLs, fingerprints, exposed paths) and + curl for a single, precise, evidence-backed request. +- Deeper crawling is [katana](katana.md); rendered interaction is + [playwright](playwright.md). diff --git a/tools/curl/client.go b/tools/curl/client.go new file mode 100644 index 00000000..7fbaf817 --- /dev/null +++ b/tools/curl/client.go @@ -0,0 +1,494 @@ +package curl + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "fmt" + "io" + "mime" + "mime/multipart" + "net" + "net/http" + "net/http/cookiejar" + "net/textproto" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + toolpb "github.com/chainreactors/aiscan/aop/tool" +) + +// A single stable, modern Chrome identity. Keeping one fingerprint per process +// (rather than rotating per request) is itself the natural shape: a real client +// does not change its User-Agent between requests from the same egress. The +// version is aligned with the uTLS HelloChrome preset used at the hub upstream +// so the header story and the (future) TLS story agree. +const chromeUserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36" + +// browserDefaults are added only when the caller has not set the same header. +// They complete the "looks like a browser navigation" shape without overriding +// anything the caller deliberately chose. Accept-Encoding is intentionally left +// to the transport (transparent gzip) so it is real rather than advertised. +var browserDefaults = []Header{ + {Name: "Accept", Value: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"}, + {Name: "Accept-Language", Value: "en-US,en;q=0.9"}, + {Name: "Sec-Ch-Ua", Value: `"Chromium";v="133", "Google Chrome";v="133", "Not(A:Brand";v="24"`}, + {Name: "Sec-Ch-Ua-Mobile", Value: "?0"}, + {Name: "Sec-Ch-Ua-Platform", Value: `"Linux"`}, + {Name: "Sec-Fetch-Dest", Value: "document"}, + {Name: "Sec-Fetch-Mode", Value: "navigate"}, + {Name: "Sec-Fetch-Site", Value: "none"}, + {Name: "Sec-Fetch-User", Value: "?1"}, + {Name: "Upgrade-Insecure-Requests", Value: "1"}, +} + +// do runs one parsed curl request end to end: builds the client (routing through +// the runner's MITM hub when the environment provides it), applies the browser +// naturalization defaults, performs the exchange, and writes curl-shaped output. +// env and workDir are per-invocation; nothing here mutates the shared Command. +func (c *Command) do(ctx context.Context, req *Request, env map[string]string, workDir string, stdout, stderr io.Writer) error { + proxyURL, caPath := c.egress(env) + if req.Proxy != "" { + // -x overrides the injected hub egress for this invocation only. + proxyURL = req.Proxy + if !strings.Contains(proxyURL, "://") { + proxyURL = "http://" + proxyURL + } + } + + client, err := c.buildClient(proxyURL, caPath, req) + if err != nil { + return err + } + + target, err := url.Parse(strings.TrimSpace(req.URL)) + if err != nil || target.Scheme == "" || target.Host == "" { + return fmt.Errorf("curl: (3) URL rejected: %s", req.URL) + } + + body, contentType, err := buildBody(req, target, workDir) + if err != nil { + return err + } + + if req.MaxTime > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, req.MaxTime) + defer cancel() + } + + httpReq, err := http.NewRequestWithContext(ctx, req.Method, target.String(), body) + if err != nil { + return fmt.Errorf("curl: %w", err) + } + applyHeaders(httpReq, req, contentType) + + if req.CookieIn != "" { + if err := seedCookies(client.Jar, target, req.CookieIn, workDir); err != nil { + return err + } + } + + if req.Verbose && !req.Silent { + writeVerboseRequest(stderr, httpReq) + } + + resp, err := client.Do(httpReq) + if err != nil { + return fmt.Errorf("curl: (7) %w", err) + } + defer resp.Body.Close() + + if req.Verbose && !req.Silent { + writeVerboseResponse(stderr, resp) + } + + out, closeOut, err := outputWriter(req, workDir, stdout) + if err != nil { + return err + } + defer closeOut() + + if req.Include { + writeStatusAndHeaders(out, resp) + } + written, err := io.Copy(out, resp.Body) + if err != nil { + return fmt.Errorf("curl: (56) %w", err) + } + + if req.CookieJar != "" { + if err := writeCookieJar(client.Jar, resp.Request.URL, resolvePath(workDir, req.CookieJar)); err != nil && !req.Silent { + c.Logger.Warnf("curl: write cookie jar: %s", err) + } + } + + if req.WriteOut != "" { + fmt.Fprint(stdout, expandWriteOut(req.WriteOut, resp, written)) + } + + c.emitArtifact(ctx, resp, written) + return nil +} + +// egress reads the hub proxy and CA path the runner injected into this +// execution's environment. The proxy URL already carries the tool-call id as +// its username, so captured flows attribute to this call; the CA is present +// only while the hub is intercepting. Falls back to the static scanner proxy. +func (c *Command) egress(env map[string]string) (proxyURL, caPath string) { + for _, key := range []string{"ALL_PROXY", "all_proxy", "HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy"} { + if v := env[key]; v != "" { + proxyURL = v + break + } + } + for _, key := range []string{"CURL_CA_BUNDLE", "SSL_CERT_FILE"} { + if v := env[key]; v != "" { + caPath = v + break + } + } + if proxyURL == "" { + proxyURL = c.Proxy + } + return proxyURL, caPath +} + +func (c *Command) buildClient(proxyURL, caPath string, req *Request) (*http.Client, error) { + tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12} + switch { + case req.Insecure: + tlsConfig.InsecureSkipVerify = true + case caPath != "": + pool, err := caPool(caPath) + if err != nil { + return nil, err + } + tlsConfig.RootCAs = pool + } + + dialTimeout := req.ConnectTimeout + if dialTimeout == 0 { + dialTimeout = 30 * time.Second + } + transport := &http.Transport{ + TLSClientConfig: tlsConfig, + DialContext: (&net.Dialer{Timeout: dialTimeout}).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: dialTimeout, + } + if proxyURL != "" { + parsed, err := url.Parse(proxyURL) + if err != nil { + return nil, fmt.Errorf("curl: invalid proxy %q: %w", proxyURL, err) + } + transport.Proxy = http.ProxyURL(parsed) + } + + jar, _ := cookiejar.New(nil) + maxRedirs := req.MaxRedirs + client := &http.Client{ + Transport: transport, + Jar: jar, + CheckRedirect: func(r *http.Request, via []*http.Request) error { + if !req.Follow { + return http.ErrUseLastResponse + } + if maxRedirs >= 0 && len(via) >= maxRedirs { + return fmt.Errorf("curl: (47) Maximum (%d) redirects followed", maxRedirs) + } + return nil + }, + } + return client, nil +} + +// caPool builds a root pool seeded from the system pool plus the hub CA, so the +// tool trusts intercepted HTTPS without losing trust in the rest of the world. +func caPool(caPath string) (*x509.CertPool, error) { + pem, err := os.ReadFile(caPath) + if err != nil { + return nil, fmt.Errorf("curl: read CA bundle: %w", err) + } + pool, err := x509.SystemCertPool() + if err != nil || pool == nil { + pool = x509.NewCertPool() + } + if !pool.AppendCertsFromPEM(pem) { + return nil, fmt.Errorf("curl: CA bundle %q contained no certificates", caPath) + } + return pool, nil +} + +// buildBody assembles the request body from -d parts or -F parts. With -G the +// data is folded into the URL query and no body is sent. Parse already rejects +// combining -d with -F and -G with -F. +func buildBody(req *Request, target *url.URL, workDir string) (io.Reader, string, error) { + if len(req.Form) > 0 { + return buildForm(req.Form, workDir) + } + if len(req.Data) == 0 { + return nil, "", nil + } + segments := make([]string, 0, len(req.Data)) + for _, part := range req.Data { + value := part.Value + if part.File { + raw, err := os.ReadFile(resolvePath(workDir, part.Value)) + if err != nil { + return nil, "", fmt.Errorf("curl: (26) Failed to open %q: %w", part.Value, err) + } + value = string(raw) + if !part.Binary && !part.URLEncode { + // Non-binary -d strips line breaks from file content, like curl. + value = strings.NewReplacer("\r", "", "\n", "").Replace(value) + } + } + if part.URLEncode { + var err error + value, err = encodeDataValue(value, part.File, workDir) + if err != nil { + return nil, "", err + } + } + segments = append(segments, value) + } + joined := strings.Join(segments, "&") + + if req.Get { + // curl appends -d data to the query verbatim (no re-encoding); encoding + // is --data-urlencode's job, applied above. + if target.RawQuery == "" { + target.RawQuery = joined + } else { + target.RawQuery += "&" + joined + } + return nil, "", nil + } + return strings.NewReader(joined), "application/x-www-form-urlencoded", nil +} + +// encodeDataValue applies --data-urlencode semantics to one part: the name +// before the first '=' stays verbatim while the content is percent-encoded; +// name@file (unreachable when File is already set) reads and encodes a file. +func encodeDataValue(value string, fromFile bool, workDir string) (string, error) { + if fromFile { + return pctEncode(value), nil + } + if name, content, ok := strings.Cut(value, "="); ok { + return name + "=" + pctEncode(content), nil + } + if name, path, ok := strings.Cut(value, "@"); ok && name != "" { + raw, err := os.ReadFile(resolvePath(workDir, path)) + if err != nil { + return "", fmt.Errorf("curl: (26) Failed to open %q: %w", path, err) + } + return name + "=" + pctEncode(string(raw)), nil + } + return pctEncode(value), nil +} + +// pctEncode percent-encodes every byte outside the RFC 3986 unreserved set, +// matching curl's --data-urlencode (space becomes %20, not +). +func pctEncode(s string) string { + const upperhex = "0123456789ABCDEF" + var sb strings.Builder + sb.Grow(len(s)) + for i := 0; i < len(s); i++ { + c := s[i] + switch { + case 'A' <= c && c <= 'Z', 'a' <= c && c <= 'z', '0' <= c && c <= '9', + c == '-', c == '_', c == '.', c == '~': + sb.WriteByte(c) + default: + sb.WriteByte('%') + sb.WriteByte(upperhex[c>>4]) + sb.WriteByte(upperhex[c&0xF]) + } + } + return sb.String() +} + +// buildForm assembles a multipart/form-data body from -F parts. +func buildForm(parts []FormPart, workDir string) (io.Reader, string, error) { + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + for _, part := range parts { + switch { + case part.File: + data, err := os.ReadFile(resolvePath(workDir, part.Value)) + if err != nil { + return nil, "", fmt.Errorf("curl: (26) Failed to open %q: %w", part.Value, err) + } + header := make(textproto.MIMEHeader) + header.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, + formQuoteEscape.Replace(part.Name), formQuoteEscape.Replace(filepath.Base(part.Value)))) + ct := part.Type + if ct == "" { + ct = mime.TypeByExtension(filepath.Ext(part.Value)) + } + if ct == "" { + ct = "application/octet-stream" + } + header.Set("Content-Type", ct) + pw, err := w.CreatePart(header) + if err != nil { + return nil, "", fmt.Errorf("curl: form part %q: %w", part.Name, err) + } + if _, err := pw.Write(data); err != nil { + return nil, "", fmt.Errorf("curl: form part %q: %w", part.Name, err) + } + case part.Content: + data, err := os.ReadFile(resolvePath(workDir, part.Value)) + if err != nil { + return nil, "", fmt.Errorf("curl: (26) Failed to open %q: %w", part.Value, err) + } + // name= %s %s %s\r\n", req.Method, req.URL.RequestURI(), req.Proto) + fmt.Fprintf(w, "> Host: %s\r\n", req.Host) + for name, values := range req.Header { + for _, v := range values { + fmt.Fprintf(w, "> %s: %s\r\n", name, v) + } + } + fmt.Fprint(w, ">\r\n") +} + +func writeVerboseResponse(w io.Writer, resp *http.Response) { + fmt.Fprintf(w, "< %s %s\r\n", resp.Proto, resp.Status) + for name, values := range resp.Header { + for _, v := range values { + fmt.Fprintf(w, "< %s: %s\r\n", name, v) + } + } + fmt.Fprint(w, "<\r\n") +} + +// expandWriteOut supports the curl -w variables the agent uses most. +func expandWriteOut(format string, resp *http.Response, size int64) string { + replacer := strings.NewReplacer( + "%{http_code}", strconv.Itoa(resp.StatusCode), + "%{response_code}", strconv.Itoa(resp.StatusCode), + "%{url_effective}", resp.Request.URL.String(), + "%{content_type}", resp.Header.Get("Content-Type"), + "%{size_download}", strconv.FormatInt(size, 10), + "\\n", "\n", "\\t", "\t", "\\r", "\r", + ) + return replacer.Replace(format) +} diff --git a/tools/curl/client_test.go b/tools/curl/client_test.go new file mode 100644 index 00000000..c08c54c5 --- /dev/null +++ b/tools/curl/client_test.go @@ -0,0 +1,328 @@ +package curl + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +// run is a small harness: parse args, execute against a real server, capture +// stdout/stderr. It never routes through a proxy (env is empty), so it exercises +// the client and naturalization logic directly. +func run(t *testing.T, args []string, env, workDir string) (stdout, stderr string, err error) { + t.Helper() + req, perr := Parse(args) + if perr != nil { + return "", "", perr + } + var out, errb strings.Builder + envMapVal := map[string]string{} + if env != "" { + k, v, _ := strings.Cut(env, "=") + envMapVal[k] = v + } + c := New() + err = c.do(context.Background(), req, envMapVal, workDir, &out, &errb) + return out.String(), errb.String(), err +} + +func TestGetWritesBody(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("hello")) + })) + defer srv.Close() + + out, _, err := run(t, []string{srv.URL}, "", "") + if err != nil { + t.Fatal(err) + } + if out != "hello" { + t.Fatalf("body = %q", out) + } +} + +func TestDefaultBrowserHeaders(t *testing.T) { + var got http.Header + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got = r.Header.Clone() + })) + defer srv.Close() + + if _, _, err := run(t, []string{srv.URL}, "", ""); err != nil { + t.Fatal(err) + } + if ua := got.Get("User-Agent"); !strings.Contains(ua, "Chrome/") { + t.Fatalf("default UA not a browser: %q", ua) + } + for _, h := range []string{"Accept", "Accept-Language", "Sec-Fetch-Mode", "Upgrade-Insecure-Requests"} { + if got.Get(h) == "" { + t.Fatalf("missing default header %s", h) + } + } +} + +func TestUserAgentOverride(t *testing.T) { + var got http.Header + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { got = r.Header.Clone() })) + defer srv.Close() + + if _, _, err := run(t, []string{"-A", "mybot/1.0", srv.URL}, "", ""); err != nil { + t.Fatal(err) + } + if got.Get("User-Agent") != "mybot/1.0" { + t.Fatalf("UA override ignored: %q", got.Get("User-Agent")) + } +} + +func TestHeaderOverrideAndRemove(t *testing.T) { + var got http.Header + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { got = r.Header.Clone() })) + defer srv.Close() + + if _, _, err := run(t, []string{"-H", "Accept: application/json", "-H", "Sec-Fetch-Mode:", srv.URL}, "", ""); err != nil { + t.Fatal(err) + } + if got.Get("Accept") != "application/json" { + t.Fatalf("Accept override ignored: %q", got.Get("Accept")) + } + if got.Get("Sec-Fetch-Mode") != "" { + t.Fatalf("Sec-Fetch-Mode should have been removed: %q", got.Get("Sec-Fetch-Mode")) + } +} + +func TestPostFormBody(t *testing.T) { + var method, body, ctype string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + method = r.Method + ctype = r.Header.Get("Content-Type") + buf := make([]byte, r.ContentLength) + r.Body.Read(buf) + body = string(buf) + })) + defer srv.Close() + + if _, _, err := run(t, []string{"-d", "a=1&b=2", srv.URL}, "", ""); err != nil { + t.Fatal(err) + } + if method != "POST" || body != "a=1&b=2" || ctype != "application/x-www-form-urlencoded" { + t.Fatalf("post wrong: %s %q %q", method, body, ctype) + } +} + +func TestGetFoldsDataIntoQuery(t *testing.T) { + var query string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { query = r.URL.RawQuery })) + defer srv.Close() + + if _, _, err := run(t, []string{"-G", "-d", "a=1&b=2", srv.URL}, "", ""); err != nil { + t.Fatal(err) + } + if query != "a=1&b=2" { + t.Fatalf("query = %q", query) + } +} + +func TestIncludeHeaders(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Test", "yes") + w.Write([]byte("body")) + })) + defer srv.Close() + + out, _, err := run(t, []string{"-i", srv.URL}, "", "") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(out, "X-Test: yes") || !strings.HasSuffix(out, "body") { + t.Fatalf("-i output wrong:\n%s", out) + } +} + +func TestNoFollowByDefault(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/a" { + http.Redirect(w, r, "/b", http.StatusFound) + return + } + w.Write([]byte("final")) + })) + defer srv.Close() + + out, _, err := run(t, []string{"-w", "%{http_code}", srv.URL + "/a"}, "", "") + if err != nil { + t.Fatal(err) + } + if !strings.HasSuffix(out, "302") { + t.Fatalf("expected 302 without -L, got %q", out) + } +} + +func TestFollowRedirect(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/a" { + http.Redirect(w, r, "/b", http.StatusFound) + return + } + w.Write([]byte("final")) + })) + defer srv.Close() + + out, _, err := run(t, []string{"-L", srv.URL + "/a"}, "", "") + if err != nil { + t.Fatal(err) + } + if out != "final" { + t.Fatalf("expected followed body, got %q", out) + } +} + +func TestOutputToFile(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("saved")) })) + defer srv.Close() + + dir := t.TempDir() + if _, _, err := run(t, []string{"-o", "resp.txt", srv.URL}, "", dir); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(dir, "resp.txt")) + if err != nil || string(data) != "saved" { + t.Fatalf("file content = %q err %v", data, err) + } +} + +func TestCookieRoundTrip(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if c, err := r.Cookie("sid"); err == nil { + w.Write([]byte("sid=" + c.Value)) + } + http.SetCookie(w, &http.Cookie{Name: "srv", Value: "1"}) + })) + defer srv.Close() + + dir := t.TempDir() + out, _, err := run(t, []string{"-b", "sid=abc", "-c", "jar.txt", srv.URL}, "", dir) + if err != nil { + t.Fatal(err) + } + if out != "sid=abc" { + t.Fatalf("cookie not sent: %q", out) + } + jar, err := os.ReadFile(filepath.Join(dir, "jar.txt")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(jar), "# Netscape HTTP Cookie File") { + t.Fatalf("jar not written in netscape format:\n%s", jar) + } +} + +func TestBasicAuth(t *testing.T) { + var user, pass string + var ok bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + user, pass, ok = r.BasicAuth() + })) + defer srv.Close() + + if _, _, err := run(t, []string{"-u", "alice:secret", srv.URL}, "", ""); err != nil { + t.Fatal(err) + } + if !ok || user != "alice" || pass != "secret" { + t.Fatalf("basic auth wrong: %v %q %q", ok, user, pass) + } +} + +func TestDataURLEncode(t *testing.T) { + var body, ctype string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + body, ctype = string(b), r.Header.Get("Content-Type") + })) + defer srv.Close() + + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "q.txt"), []byte("x y\n"), 0o644); err != nil { + t.Fatal(err) + } + _, _, err := run(t, []string{ + "--data-urlencode", "q=a b&c=d", + "--data-urlencode", "file@q.txt", + "--data-urlencode", "plain", + srv.URL, + }, "", dir) + if err != nil { + t.Fatal(err) + } + want := "q=a%20b%26c%3Dd&file=x%20y%0A&plain" + if body != want { + t.Fatalf("body = %q, want %q", body, want) + } + if ctype != "application/x-www-form-urlencoded" { + t.Fatalf("content-type = %q", ctype) + } +} + +func TestMultipartForm(t *testing.T) { + var field, fileName, fileContent, fileType, ctype string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctype = r.Header.Get("Content-Type") + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Errorf("parse multipart: %v", err) + return + } + field = r.FormValue("field") + f, fh, err := r.FormFile("up") + if err != nil { + t.Errorf("form file: %v", err) + return + } + defer f.Close() + b, _ := io.ReadAll(f) + fileName, fileContent, fileType = fh.Filename, string(b), fh.Header.Get("Content-Type") + })) + defer srv.Close() + + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "a.txt"), []byte("file-bytes"), 0o644); err != nil { + t.Fatal(err) + } + _, _, err := run(t, []string{"-F", "field=value", "-F", "up=@a.txt;type=text/plain", srv.URL}, "", dir) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(ctype, "multipart/form-data; boundary=") { + t.Fatalf("content-type = %q", ctype) + } + if field != "value" || fileName != "a.txt" || fileContent != "file-bytes" || fileType != "text/plain" { + t.Fatalf("multipart wrong: field=%q file=%q %q %q", field, fileName, fileContent, fileType) + } +} + +func TestProxyOverride(t *testing.T) { + // A stand-in proxy answers directly, so a response proves -x steered the + // request away from the target. + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.HasPrefix(r.RequestURI, "http://") { + t.Errorf("proxy got non-absolute URI %q", r.RequestURI) + } + w.Write([]byte("via-proxy")) + })) + defer proxy.Close() + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("direct")) + })) + defer target.Close() + + out, _, err := run(t, []string{"-x", proxy.URL, target.URL}, "", "") + if err != nil { + t.Fatal(err) + } + if out != "via-proxy" { + t.Fatalf("body = %q, want via-proxy", out) + } +} diff --git a/tools/curl/cookies.go b/tools/curl/cookies.go new file mode 100644 index 00000000..2d6e9c28 --- /dev/null +++ b/tools/curl/cookies.go @@ -0,0 +1,93 @@ +package curl + +import ( + "bufio" + "fmt" + "net/http" + "net/url" + "os" + "strings" +) + +// seedCookies loads cookies into the jar before the request. Following curl, a +// value containing '=' is an inline cookie string; anything else is a file +// (Netscape format, or plain name=value lines). +func seedCookies(jar http.CookieJar, target *url.URL, spec, workDir string) error { + if jar == nil { + return nil + } + if strings.Contains(spec, "=") { + jar.SetCookies(target, parseCookieString(spec)) + return nil + } + file, err := os.Open(resolvePath(workDir, spec)) + if err != nil { + return fmt.Errorf("curl: (25) cannot open cookie file %q: %w", spec, err) + } + defer file.Close() + + var inline []*http.Cookie + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || (strings.HasPrefix(line, "#") && !strings.HasPrefix(line, "#HttpOnly_")) { + continue + } + line = strings.TrimPrefix(line, "#HttpOnly_") + if fields := strings.Split(line, "\t"); len(fields) == 7 { + // domain, flag, path, secure, expiration, name, value + jar.SetCookies(cookieURL(fields[0], fields[2], target), []*http.Cookie{{Name: fields[5], Value: fields[6]}}) + continue + } + inline = append(inline, parseCookieString(line)...) + } + if len(inline) > 0 { + jar.SetCookies(target, inline) + } + return scanner.Err() +} + +func parseCookieString(spec string) []*http.Cookie { + var cookies []*http.Cookie + for _, pair := range strings.Split(spec, ";") { + name, value, ok := strings.Cut(strings.TrimSpace(pair), "=") + if !ok || name == "" { + continue + } + cookies = append(cookies, &http.Cookie{Name: strings.TrimSpace(name), Value: strings.TrimSpace(value)}) + } + return cookies +} + +func cookieURL(domain, path string, fallback *url.URL) *url.URL { + domain = strings.TrimPrefix(domain, ".") + if domain == "" { + return fallback + } + if path == "" { + path = "/" + } + return &url.URL{Scheme: "https", Host: domain, Path: path} +} + +// writeCookieJar persists the jar's cookies for the final URL in Netscape +// format, the shape curl's -c produces and -b consumes. http.CookieJar exposes +// only name/value, so domain/path are taken from the request URL. +func writeCookieJar(jar http.CookieJar, u *url.URL, path string) error { + if jar == nil || u == nil { + return nil + } + file, err := os.Create(path) + if err != nil { + return err + } + defer file.Close() + + fmt.Fprintln(file, "# Netscape HTTP Cookie File") + fmt.Fprintln(file, "# This file was generated by aiscan curl. Edit at your own risk.") + fmt.Fprintln(file) + for _, cookie := range jar.Cookies(u) { + fmt.Fprintf(file, "%s\tFALSE\t/\tFALSE\t0\t%s\t%s\n", u.Hostname(), cookie.Name, cookie.Value) + } + return nil +} diff --git a/tools/curl/curl.go b/tools/curl/curl.go new file mode 100644 index 00000000..23d1ae04 --- /dev/null +++ b/tools/curl/curl.go @@ -0,0 +1,117 @@ +package curl + +import ( + "context" + "strings" + + aop "github.com/chainreactors/aiscan/aop" + "github.com/chainreactors/aiscan/core/telemetry" + coretool "github.com/chainreactors/aiscan/core/tool" + "github.com/chainreactors/aiscan/pkg/commands" + "github.com/chainreactors/aiscan/tools/toolargs" +) + +// Command is a pure-Go, evidence-first reimplementation of curl. It exposes a +// curl-shaped flag surface so the agent uses it exactly as it would use system +// curl (this command shadows the system binary), while every request routes +// through the runner's MITM hub — attributed by tool-call id and captured as +// http.exchange evidence — and carries a browser-shaped header set by default +// instead of announcing itself as a scanner. +type Command struct { + toolargs.Base +} + +func New() *Command { + c := &Command{} + c.InitLogger(nil) + return c +} + +func (c *Command) WithLogger(logger telemetry.Logger) *Command { + c.InitLogger(logger) + return c +} + +func (c *Command) WithProxy(proxy string) *Command { + c.Proxy = proxy + return c +} + +func (c *Command) WithEvents(events aop.EventEmitter) *Command { + c.Events = events + return c +} + +func (c *Command) Name() string { return "curl" } + +func (c *Command) Usage() string { + return `curl - transfer a URL (pure-Go, browser-naturalized, evidence-first) + +Usage: curl [options] + +Supported options: + -X, --request HTTP method + -H, --header Extra header ("Name: value"); repeatable + -d, --data POST body (@file to read a file); --data-raw / --data-binary + --data-urlencode Like -d, but percent-encode the content (name=content, name@file) + -F, --form Multipart form field; name=@file uploads a file (;type=mime), name= Cookie string ("k=v") or file to read + -c, --cookie-jar Write cookies to this file after the exchange + -L, --location Follow redirects (--max-redirs ) + -A, --user-agent Override the User-Agent + -e, --referer Referer header + -u, --user Basic authentication + -o, --output Write body to file instead of stdout + -i, --include Include response headers in the output + -s, --silent Silent mode + -S, --show-error Show errors even with -s + -w, --write-out After completion, print %{http_code}, %{url_effective}, ... + -v, --verbose Log request/response headers + -k, --insecure Do not verify TLS + -x, --proxy Use this proxy instead of the runner egress + --connect-timeout Connection timeout, seconds + --max-time Overall timeout, seconds + +Unlisted flags are rejected rather than silently ignored. Requests are routed +through the runner proxy and recorded as HTTP evidence; a browser User-Agent and +header set are applied unless you override them.` +} + +func (c *Command) QuickReference() string { + return `### curl — HTTP requests (pure-Go, browser-naturalized, evidence-first) + curl GET a URL + curl -X POST -d 'a=1' POST form data + curl -H 'Authorization: ...' + curl -i -L Include headers, follow redirects + curl -b 'sid=abc' -c jar.txt Send and persist cookies + curl -F 'file=@a.png' Multipart form upload` +} + +// Run parses the curl-shaped argument vector and performs one exchange. The +// proxy and CA that route/trust the MITM hub arrive in execution.Env (the +// builtin runs in-process and does not inherit them from os.Environ). +func (c *Command) Run(ctx context.Context, execution *commands.Execution) (_ any, err error) { + defer telemetry.RecoverAsError("curl", &err) + + req, err := Parse(execution.Args) + if err != nil { + return nil, err + } + + workDir := execution.Dir + if workDir == "" { + workDir = coretool.WorkDirFromContext(ctx, c.WorkDir) + } + return nil, c.do(ctx, req, envMap(execution.Env), workDir, execution.Stdout, execution.Stderr) +} + +func envMap(env []string) map[string]string { + values := make(map[string]string, len(env)) + for _, item := range env { + if key, value, ok := strings.Cut(item, "="); ok { + values[key] = value + } + } + return values +} diff --git a/tools/curl/parse.go b/tools/curl/parse.go new file mode 100644 index 00000000..be441664 --- /dev/null +++ b/tools/curl/parse.go @@ -0,0 +1,393 @@ +package curl + +import ( + "fmt" + "strconv" + "strings" + "time" +) + +// Header is one -H value. Value is empty for a "Name;" send-empty form; Remove +// is set for a "Name:" form that suppresses a default header. +type Header struct { + Name string + Value string + Remove bool +} + +// DataPart is one -d/--data family value. File means the value is a path to +// read (@file); Binary keeps the bytes verbatim (--data-binary) while the +// default -d strips CR/LF from file contents; Raw disables @file handling +// (--data-raw); URLEncode percent-encodes the content (--data-urlencode). +type DataPart struct { + Value string + File bool + Binary bool + Raw bool + URLEncode bool +} + +// FormPart is one -F/--form value. File means Value is a path whose bytes +// become a file part (name=@path, optional ;type= MIME override); Content +// means Value is a path whose text becomes the field value (name= 0 { + return nil, fmt.Errorf("curl: --url and a positional URL cannot both be given") + } + + if r.Method == "" { + if (len(r.Data) > 0 || len(r.Form) > 0) && !r.Get { + r.Method = "POST" + } else { + r.Method = "GET" + } + } + if len(r.Form) > 0 && len(r.Data) > 0 { + return nil, fmt.Errorf("curl: (2) -d/--data and -F/--form cannot be combined") + } + if len(r.Form) > 0 && r.Get { + return nil, fmt.Errorf("curl: (2) -G cannot be used with -F/--form") + } + return r, nil +} + +func (r *Request) applyShortBundle(bundle string, args []string, i *int) error { + for j := 0; j < len(bundle); j++ { + ch := bundle[j] + if long, ok := boolShort[ch]; ok { + if err := r.applyLong(long, func() (string, error) { + return "", fmt.Errorf("curl: -%c takes no value", ch) + }); err != nil { + return err + } + continue + } + if long, ok := valueShort[ch]; ok { + // Value is the rest of the bundle when non-empty, else the next token. + var value string + if j+1 < len(bundle) { + value = bundle[j+1:] + } else { + if *i >= len(args) { + return fmt.Errorf("curl: option -%c requires a value", ch) + } + value = args[*i] + *i++ + } + return r.applyLong(long, func() (string, error) { return value, nil }) + } + return fmt.Errorf("curl: unsupported flag -%c", ch) + } + return nil +} + +func (r *Request) applyLong(name string, need func() (string, error)) error { + switch name { + case "url": + v, err := need() + if err != nil { + return err + } + r.URL = v + case "request": + v, err := need() + if err != nil { + return err + } + r.Method = strings.ToUpper(v) + case "header": + v, err := need() + if err != nil { + return err + } + r.Headers = append(r.Headers, parseHeader(v)) + case "data", "data-ascii": + return r.addData(need, DataPart{}) + case "data-raw": + return r.addData(need, DataPart{Raw: true}) + case "data-binary": + return r.addData(need, DataPart{Binary: true}) + case "data-urlencode": + return r.addData(need, DataPart{URLEncode: true}) + case "form": + v, err := need() + if err != nil { + return err + } + part, err := parseFormPart(v) + if err != nil { + return err + } + r.Form = append(r.Form, part) + case "get": + r.Get = true + case "location": + r.Follow = true + case "max-redirs": + v, err := need() + if err != nil { + return err + } + n, err := strconv.Atoi(strings.TrimSpace(v)) + if err != nil { + return fmt.Errorf("curl: --max-redirs expects an integer: %q", v) + } + r.MaxRedirs = n + case "user-agent": + v, err := need() + if err != nil { + return err + } + r.UserAgent = v + case "referer": + v, err := need() + if err != nil { + return err + } + r.Referer = v + case "user": + v, err := need() + if err != nil { + return err + } + r.User = v + case "cookie": + v, err := need() + if err != nil { + return err + } + r.CookieIn = v + case "cookie-jar": + v, err := need() + if err != nil { + return err + } + r.CookieJar = v + case "output": + v, err := need() + if err != nil { + return err + } + r.Output = v + case "include": + r.Include = true + case "silent": + r.Silent = true + case "show-error": + r.ShowError = true + case "write-out": + v, err := need() + if err != nil { + return err + } + r.WriteOut = v + case "verbose": + r.Verbose = true + case "insecure": + r.Insecure = true + case "proxy": + v, err := need() + if err != nil { + return err + } + r.Proxy = v + case "connect-timeout": + v, err := need() + if err != nil { + return err + } + d, err := parseSeconds(v) + if err != nil { + return fmt.Errorf("curl: --connect-timeout %w", err) + } + r.ConnectTimeout = d + case "max-time": + v, err := need() + if err != nil { + return err + } + d, err := parseSeconds(v) + if err != nil { + return fmt.Errorf("curl: --max-time %w", err) + } + r.MaxTime = d + default: + return fmt.Errorf("curl: unsupported flag --%s", name) + } + return nil +} + +func (r *Request) addData(need func() (string, error), tpl DataPart) error { + v, err := need() + if err != nil { + return err + } + part := tpl + if !tpl.Raw && strings.HasPrefix(v, "@") { + part.File = true + part.Value = v[1:] + } else { + part.Value = v + } + r.Data = append(r.Data, part) + return nil +} + +// parseFormPart splits a -F value. curl requires name=content; @path attaches +// a file (with an optional ;type= MIME override), = 0 { + name := strings.TrimSpace(raw[:idx]) + value := strings.TrimSpace(raw[idx+1:]) + if value == "" { + return Header{Name: name, Remove: true} + } + return Header{Name: name, Value: value} + } + if idx := strings.IndexByte(raw, ';'); idx >= 0 { + return Header{Name: strings.TrimSpace(raw[:idx]), Value: ""} + } + return Header{Name: strings.TrimSpace(raw)} +} + +func parseSeconds(v string) (time.Duration, error) { + f, err := strconv.ParseFloat(strings.TrimSpace(v), 64) + if err != nil || f < 0 { + return 0, fmt.Errorf("expects a non-negative number of seconds: %q", v) + } + return time.Duration(f * float64(time.Second)), nil +} diff --git a/tools/curl/parse_test.go b/tools/curl/parse_test.go new file mode 100644 index 00000000..40017e3f --- /dev/null +++ b/tools/curl/parse_test.go @@ -0,0 +1,194 @@ +package curl + +import ( + "testing" + "time" +) + +func TestParseBasicGet(t *testing.T) { + req, err := Parse([]string{"https://example.com/x"}) + if err != nil { + t.Fatal(err) + } + if req.URL != "https://example.com/x" || req.Method != "GET" { + t.Fatalf("got %q %q", req.Method, req.URL) + } +} + +func TestParseDataImpliesPost(t *testing.T) { + req, err := Parse([]string{"-d", "a=1", "https://x"}) + if err != nil { + t.Fatal(err) + } + if req.Method != "POST" { + t.Fatalf("expected POST, got %q", req.Method) + } + if len(req.Data) != 1 || req.Data[0].Value != "a=1" { + t.Fatalf("bad data: %+v", req.Data) + } +} + +func TestParseExplicitMethodWins(t *testing.T) { + req, err := Parse([]string{"-XPUT", "-d", "a=1", "https://x"}) + if err != nil { + t.Fatal(err) + } + if req.Method != "PUT" { + t.Fatalf("expected PUT, got %q", req.Method) + } +} + +func TestParseRepeatedHeaders(t *testing.T) { + req, err := Parse([]string{"-H", "A: 1", "-H", "B: 2", "-H", "C;", "-H", "D:", "https://x"}) + if err != nil { + t.Fatal(err) + } + if len(req.Headers) != 4 { + t.Fatalf("want 4 headers, got %d", len(req.Headers)) + } + if req.Headers[2].Name != "C" || req.Headers[2].Value != "" || req.Headers[2].Remove { + t.Fatalf("send-empty form wrong: %+v", req.Headers[2]) + } + if !req.Headers[3].Remove { + t.Fatalf("remove form wrong: %+v", req.Headers[3]) + } +} + +func TestParseShortBundle(t *testing.T) { + req, err := Parse([]string{"-sSL", "https://x"}) + if err != nil { + t.Fatal(err) + } + if !req.Silent || !req.ShowError || !req.Follow { + t.Fatalf("bundle not applied: %+v", req) + } +} + +func TestParseBundleTrailingValue(t *testing.T) { + req, err := Parse([]string{"-so", "out.txt", "https://x"}) + if err != nil { + t.Fatal(err) + } + if !req.Silent || req.Output != "out.txt" { + t.Fatalf("bundle trailing value wrong: %+v", req) + } +} + +func TestParseDataFileAndBinary(t *testing.T) { + req, err := Parse([]string{"--data-binary", "@body.bin", "--data-raw", "@literal", "https://x"}) + if err != nil { + t.Fatal(err) + } + if !req.Data[0].File || !req.Data[0].Binary { + t.Fatalf("data-binary @file wrong: %+v", req.Data[0]) + } + if req.Data[1].File || !req.Data[1].Raw || req.Data[1].Value != "@literal" { + t.Fatalf("data-raw should keep @ literal: %+v", req.Data[1]) + } +} + +func TestParseTimeouts(t *testing.T) { + req, err := Parse([]string{"--connect-timeout", "2.5", "--max-time=10", "https://x"}) + if err != nil { + t.Fatal(err) + } + if req.ConnectTimeout != 2500*time.Millisecond || req.MaxTime != 10*time.Second { + t.Fatalf("timeouts wrong: %v %v", req.ConnectTimeout, req.MaxTime) + } +} + +func TestParseUnsupportedFlagErrors(t *testing.T) { + if _, err := Parse([]string{"--http2-prior-knowledge", "https://x"}); err == nil { + t.Fatal("expected error for unsupported long flag") + } + if _, err := Parse([]string{"-Z", "https://x"}); err == nil { + t.Fatal("expected error for unsupported short flag") + } +} + +func TestParseNoURL(t *testing.T) { + if _, err := Parse([]string{"-s"}); err == nil { + t.Fatal("expected error when no URL is given") + } +} + +func TestParseURLFlag(t *testing.T) { + req, err := Parse([]string{"--url", "https://x/y", "-G", "-d", "a=1"}) + if err != nil { + t.Fatal(err) + } + if req.URL != "https://x/y" || !req.Get || req.Method != "GET" { + t.Fatalf("url flag / -G wrong: %+v", req) + } +} + +func TestParseMaxRedirsDefault(t *testing.T) { + req, _ := Parse([]string{"-L", "https://x"}) + if req.MaxRedirs != 50 { + t.Fatalf("default max-redirs should be 50, got %d", req.MaxRedirs) + } +} + +func TestParseForm(t *testing.T) { + req, err := Parse([]string{"-F", "field=value", "-F", "up=@a.txt;type=text/plain", "-F", "note= maxBodySnip { - t.Fatalf("body snip = %d, want <= %d", len(f.ResponseBodySnip), maxBodySnip) + if len(f.Response.Body) > maxBodySnip { + t.Fatalf("body snip = %d, want <= %d", len(f.Response.Body), maxBodySnip) } } } diff --git a/tools/proxy/hub_traffic.go b/tools/proxy/hub_traffic.go index 0ba85189..4066103f 100644 --- a/tools/proxy/hub_traffic.go +++ b/tools/proxy/hub_traffic.go @@ -1,8 +1,6 @@ package proxy import ( - "net/http" - "strconv" "sync" traffic "github.com/chainreactors/aiscan/aop/traffic" @@ -66,35 +64,17 @@ func (h *ProxyHub) Subscribe(buffer int) (<-chan *traffic.Flow, func()) { return channel, cancel } +// flowToProto renders a stored flow as a wire Flow: the exchange semantics go +// through the canonical Exchange, attribution (tool id, timestamp) is stamped +// on top. func flowToProto(flow *Flow) *traffic.Flow { if flow == nil { return nil } - message := &traffic.Flow{ - Id: strconv.Itoa(flow.ID), - ToolId: flow.ToolID, - Method: flow.Method, - Url: flow.URL, - StatusCode: int32(flow.StatusCode), - RequestHeaders: headersToProto(flow.RequestHeaders), - ResponseHeaders: headersToProto(flow.ResponseHeaders), - RequestBody: flow.RequestBodySnip, - ResponseBody: flow.ResponseBodySnip, - Error: flow.Error, - Complete: flow.Error == "" && flow.StatusCode != 0, - } + message := flow.Exchange.Proto() + message.ToolId = flow.ToolID if !flow.Timestamp.IsZero() { message.Timestamp = timestamppb.New(flow.Timestamp) } return message } - -func headersToProto(headers http.Header) []*traffic.Header { - var result []*traffic.Header - for name, values := range headers { - for _, value := range values { - result = append(result, &traffic.Header{Name: name, Value: value}) - } - } - return result -} diff --git a/tools/proxy/mitm.go b/tools/proxy/mitm.go index 392e9a48..26ac1cea 100644 --- a/tools/proxy/mitm.go +++ b/tools/proxy/mitm.go @@ -4,11 +4,13 @@ import ( "context" "fmt" "net/http" + "sort" "strconv" "strings" "sync" "time" + traffic "github.com/chainreactors/aiscan/aop/traffic" "github.com/chainreactors/aiscan/core/telemetry" "github.com/chainreactors/aiscan/pkg/commands" mitmproxy "github.com/chainreactors/utils/mitmproxy/proxy" @@ -196,25 +198,33 @@ func (a *captureAddon) Response(f *mitmproxy.Flow) { } } flow := Flow{ - Timestamp: f.StartTime, - ToolID: toolIDOf(f), - Method: f.Request.Method, - URL: f.Request.URL.String(), - Host: f.Request.URL.Hostname(), - Duration: dur, - TLS: f.ConnContext.ClientConn.Tls, - RequestHeaders: f.Request.Header.Clone(), + Exchange: traffic.Exchange{ + Request: traffic.Request{ + Method: f.Request.Method, + URL: f.Request.URL.String(), + Protocol: f.Request.Proto, + Headers: pairsFromHTTP(f.Request.Header), + }, + }, + Timestamp: f.StartTime, + ToolID: toolIDOf(f), + Host: f.Request.URL.Hostname(), + Duration: dur, + TLS: f.ConnContext.ClientConn.Tls, } if len(f.Request.Body) > 0 { - flow.RequestBodySnip = snip(f.Request.Body, maxBodySnip) + flow.Request.Body = snip(f.Request.Body, maxBodySnip) } if f.Response != nil { - flow.StatusCode = f.Response.StatusCode - flow.ResponseHeaders = f.Response.Header.Clone() + flow.Exchange.Response = &traffic.Response{ + StatusCode: f.Response.StatusCode, + Headers: pairsFromHTTP(f.Response.Header), + } flow.ContentType = f.Response.Header.Get("Content-Type") if len(f.Response.Body) > 0 { - flow.ResponseBodySnip = snip(f.Response.Body, maxBodySnip) + flow.Response.Body = snip(f.Response.Body, maxBodySnip) } + flow.Complete = f.Response.StatusCode != 0 } a.hub.ingest(flow) } @@ -227,13 +237,18 @@ func (a *captureAddon) RequestError(f *mitmproxy.Flow, err error) { } } a.hub.ingest(Flow{ + Exchange: traffic.Exchange{ + Request: traffic.Request{ + Method: f.Request.Method, + URL: f.Request.URL.String(), + Protocol: f.Request.Proto, + }, + Error: err.Error(), + }, Timestamp: f.StartTime, ToolID: toolIDOf(f), - Method: f.Request.Method, - URL: f.Request.URL.String(), Host: f.Request.URL.Hostname(), Duration: dur, - Error: err.Error(), }) } @@ -246,26 +261,42 @@ func snip(b []byte, max int) []byte { return out } +// pairsFromHTTP flattens an http.Header into the canonical pair sequence. The +// wire order is already lost inside net/http, so names are sorted to keep the +// stored form deterministic. +func pairsFromHTTP(headers http.Header) []traffic.Pair { + if len(headers) == 0 { + return nil + } + names := make([]string, 0, len(headers)) + for name := range headers { + names = append(names, name) + } + sort.Strings(names) + out := make([]traffic.Pair, 0, len(headers)) + for _, name := range names { + for _, value := range headers[name] { + out = append(out, traffic.Pair{Name: name, Value: value}) + } + } + return out +} + // --------------------------------------------------------------------------- // Flow + FlowStore // --------------------------------------------------------------------------- +// Flow is the hub's stored capture: the canonical exchange plus the hub-only +// metadata (attribution, timing, TLS) the mitm query verbs filter and format +// on. The wire view is Exchange.Proto with ToolID/Timestamp stamped. type Flow struct { - ID int - ToolID string - Timestamp time.Time - Method string - URL string - Host string - StatusCode int - ContentType string - Duration time.Duration - RequestHeaders http.Header - RequestBodySnip []byte - ResponseHeaders http.Header - ResponseBodySnip []byte - TLS bool - Error string + traffic.Exchange + ToolID string + Timestamp time.Time + Host string + ContentType string + Duration time.Duration + TLS bool } type QueryOpts struct { @@ -295,7 +326,7 @@ func (s *FlowStore) Add(f Flow) Flow { s.mu.Lock() defer s.mu.Unlock() s.seq++ - f.ID = s.seq + f.Exchange.ID = strconv.Itoa(s.seq) if len(s.flows) >= s.cap { copy(s.flows, s.flows[1:]) s.flows[len(s.flows)-1] = f @@ -314,8 +345,10 @@ func (s *FlowStore) Query(opts QueryOpts) []Flow { if opts.Host != "" && !strings.Contains(strings.ToLower(f.Host), strings.ToLower(opts.Host)) { continue } - if opts.Status != "" && !matchStatus(f.StatusCode, opts.Status) { - continue + if opts.Status != "" { + if f.Response == nil || !matchStatus(f.Response.StatusCode, opts.Status) { + continue + } } if opts.CType != "" && !strings.Contains(strings.ToLower(f.ContentType), strings.ToLower(opts.CType)) { continue @@ -331,8 +364,9 @@ func (s *FlowStore) Query(opts QueryOpts) []Flow { func (s *FlowStore) Get(id int) *Flow { s.mu.RLock() defer s.mu.RUnlock() + want := strconv.Itoa(id) for i := range s.flows { - if s.flows[i].ID == id { + if s.flows[i].Exchange.ID == want { f := s.flows[i] return &f } @@ -391,7 +425,7 @@ func formatFlowList(flows []Flow) string { if idx := strings.Index(ct, ";"); idx > 0 { ct = ct[:idx] } - urlStr := f.URL + urlStr := f.Request.URL if len(urlStr) > 50 { urlStr = urlStr[:47] + "..." } @@ -399,30 +433,40 @@ func formatFlowList(flows []Flow) string { if f.Error != "" { errMark = " ERR" } - sb.WriteString(fmt.Sprintf(" %-6d %-6s %-4d %-50s %-14s %dms%s\n", - f.ID, f.Method, f.StatusCode, urlStr, truncate(ct, 14), f.Duration.Milliseconds(), errMark)) + sb.WriteString(fmt.Sprintf(" %-6s %-6s %-4d %-50s %-14s %dms%s\n", + f.Exchange.ID, f.Request.Method, statusCodeOf(&f), urlStr, truncate(ct, 14), f.Duration.Milliseconds(), errMark)) } return sb.String() } +// statusCodeOf reports the response status, 0 for a request-only flow. +func statusCodeOf(f *Flow) int { + if f.Response == nil { + return 0 + } + return f.Response.StatusCode +} + func formatFlowDetail(f *Flow) string { var sb strings.Builder - sb.WriteString(fmt.Sprintf("=== Flow #%d ===\n", f.ID)) + sb.WriteString(fmt.Sprintf("=== Flow #%s ===\n", f.Exchange.ID)) sb.WriteString(fmt.Sprintf("Time: %s Method: %s Status: %d Duration: %dms TLS: %v\n", - f.Timestamp.Format(time.RFC3339), f.Method, f.StatusCode, f.Duration.Milliseconds(), f.TLS)) - sb.WriteString(fmt.Sprintf("URL: %s\n", f.URL)) + f.Timestamp.Format(time.RFC3339), f.Request.Method, statusCodeOf(f), f.Duration.Milliseconds(), f.TLS)) + sb.WriteString(fmt.Sprintf("URL: %s\n", f.Request.URL)) if f.Error != "" { sb.WriteString(fmt.Sprintf("Error: %s\n", f.Error)) } sb.WriteString("\n--- Request Headers ---\n") - writeHeaders(&sb, f.RequestHeaders) - if len(f.RequestBodySnip) > 0 { - sb.WriteString(fmt.Sprintf("\n--- Request Body (%d bytes) ---\n%s\n", len(f.RequestBodySnip), f.RequestBodySnip)) + writeHeaders(&sb, f.Request.Headers) + if len(f.Request.Body) > 0 { + sb.WriteString(fmt.Sprintf("\n--- Request Body (%d bytes) ---\n%s\n", len(f.Request.Body), f.Request.Body)) } - sb.WriteString("\n--- Response Headers ---\n") - writeHeaders(&sb, f.ResponseHeaders) - if len(f.ResponseBodySnip) > 0 { - sb.WriteString(fmt.Sprintf("\n--- Response Body (%d bytes) ---\n%s\n", len(f.ResponseBodySnip), f.ResponseBodySnip)) + if f.Response != nil { + sb.WriteString("\n--- Response Headers ---\n") + writeHeaders(&sb, f.Response.Headers) + if len(f.Response.Body) > 0 { + sb.WriteString(fmt.Sprintf("\n--- Response Body (%d bytes) ---\n%s\n", len(f.Response.Body), f.Response.Body)) + } } return sb.String() } @@ -439,7 +483,7 @@ func formatFlowAnalysis(flows []Flow) string { var errCount int for _, f := range flows { hostCounts[f.Host]++ - statusCounts[f.StatusCode/100]++ + statusCounts[statusCodeOf(&f)/100]++ if f.Error != "" { errCount++ } @@ -454,12 +498,12 @@ func formatFlowAnalysis(flows []Flow) string { sb.WriteString("\n\n") for _, f := range flows { - sb.WriteString(fmt.Sprintf("#%d [%d] %s %s (%dms)\n", f.ID, f.StatusCode, f.Method, f.URL, f.Duration.Milliseconds())) + sb.WriteString(fmt.Sprintf("#%s [%d] %s %s (%dms)\n", f.Exchange.ID, statusCodeOf(&f), f.Request.Method, f.Request.URL, f.Duration.Milliseconds())) if f.Error != "" { sb.WriteString(fmt.Sprintf(" ERROR: %s\n", f.Error)) } - if len(f.ResponseBodySnip) > 0 { - body := string(f.ResponseBodySnip) + if f.Response != nil && len(f.Response.Body) > 0 { + body := string(f.Response.Body) if len(body) > 500 { body = body[:500] + "..." } @@ -469,10 +513,8 @@ func formatFlowAnalysis(flows []Flow) string { return sb.String() } -func writeHeaders(sb *strings.Builder, h http.Header) { - for k, vals := range h { - for _, v := range vals { - sb.WriteString(fmt.Sprintf(" %s: %s\n", k, v)) - } +func writeHeaders(sb *strings.Builder, headers []traffic.Pair) { + for _, p := range headers { + sb.WriteString(fmt.Sprintf(" %s: %s\n", p.Name, p.Value)) } } diff --git a/tools/proxy/mitm_test.go b/tools/proxy/mitm_test.go index e051e947..4f46bc66 100644 --- a/tools/proxy/mitm_test.go +++ b/tools/proxy/mitm_test.go @@ -15,6 +15,7 @@ import ( "testing" "time" + traffic "github.com/chainreactors/aiscan/aop/traffic" "github.com/chainreactors/proxyclient" mitmproxy "github.com/chainreactors/utils/mitmproxy/proxy" ) @@ -88,8 +89,8 @@ func TestMITMCapture_HTTP(t *testing.T) { t.Fatalf("expected 1 flow, got %d", store.Count()) } f := store.Get(1) - if f.StatusCode != 200 { - t.Fatalf("captured flow status %d, want 200", f.StatusCode) + if f.Response.StatusCode != 200 { + t.Fatalf("captured flow status %d, want 200", f.Response.StatusCode) } } @@ -399,7 +400,13 @@ func TestMITMThroughput(t *testing.T) { func BenchmarkFlowStore_Add(b *testing.B) { store := NewFlowStore(10000) - f := Flow{Method: "GET", URL: "http://example.com/", StatusCode: 200, Host: "example.com"} + f := Flow{ + Exchange: traffic.Exchange{ + Request: traffic.Request{Method: "GET", URL: "http://example.com/"}, + Response: &traffic.Response{StatusCode: 200}, + }, + Host: "example.com", + } b.ResetTimer() for i := 0; i < b.N; i++ { store.Add(f) @@ -410,10 +417,11 @@ func BenchmarkFlowStore_Query(b *testing.B) { store := NewFlowStore(10000) for i := 0; i < 10000; i++ { store.Add(Flow{ - Method: "GET", - URL: fmt.Sprintf("http://host%d.com/path%d", i%10, i), - StatusCode: 200 + (i%5)*100, - Host: fmt.Sprintf("host%d.com", i%10), + Exchange: traffic.Exchange{ + Request: traffic.Request{Method: "GET", URL: fmt.Sprintf("http://host%d.com/path%d", i%10, i)}, + Response: &traffic.Response{StatusCode: 200 + (i%5)*100}, + }, + Host: fmt.Sprintf("host%d.com", i%10), }) } opts := QueryOpts{Host: "host5", Status: "2xx", Last: 20} @@ -433,14 +441,20 @@ func TestFlowStoreMemory(t *testing.T) { store := NewFlowStore(10000) for i := 0; i < 10000; i++ { store.Add(Flow{ - Method: "GET", - URL: fmt.Sprintf("http://example.com/path/%d", i), - StatusCode: 200, - Host: "example.com", - ContentType: "text/html", - RequestHeaders: http.Header{"User-Agent": {"test"}}, - ResponseHeaders: http.Header{"Content-Type": {"text/html"}}, - ResponseBodySnip: make([]byte, 4096), + Exchange: traffic.Exchange{ + Request: traffic.Request{ + Method: "GET", + URL: fmt.Sprintf("http://example.com/path/%d", i), + Headers: []traffic.Pair{{Name: "User-Agent", Value: "test"}}, + }, + Response: &traffic.Response{ + StatusCode: 200, + Headers: []traffic.Pair{{Name: "Content-Type", Value: "text/html"}}, + Body: make([]byte, 4096), + }, + }, + Host: "example.com", + ContentType: "text/html", }) } diff --git a/web/frontend/cyber-ui b/web/frontend/cyber-ui index de122ac9..919b55d8 160000 --- a/web/frontend/cyber-ui +++ b/web/frontend/cyber-ui @@ -1 +1 @@ -Subproject commit de122ac9009328959774aa29b3b7a3fe1b2890b1 +Subproject commit 919b55d8a407f54f15c73dfc6c59d02dacf925fe