Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

callback addon: support intercepting requests #105

Merged
merged 4 commits into from
Jul 5, 2024
Merged

callback addon: support intercepting requests #105

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
de656f0
callback: split API shape into request/response
kegsay Jul 4, 2024
26941ae
Refactor CallbackServer to support 2 kinds of handlers
kegsay Jul 5, 2024
005bb91
Get request callback interception working with the callback server
kegsay Jul 5, 2024
7b50a78
Update tests/mitmproxy_addons/callback.py
kegsay Jul 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion internal/deploy/callback_addon_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@ func TestCallbackAddon(t *testing.T) {
mitmClient := deployment.MITM()
mitmOpts := map[string]any{
"callback": map[string]any{
"callback_url": callbackURL,
"callback_response_url": callbackURL,
},
}
if tc.filter != "" {
Expand Down
4 changes: 2 additions & 2 deletions internal/deploy/mitm.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,8 +126,8 @@ func (c *MITMConfiguration) Execute(inner func()) {
defer closeCallbackServer()

body["callback"] = map[string]any{
"callback_url": callbackURL,
"filter": pathConfig.filter(),
"callback_response_url": callbackURL,
"filter": pathConfig.filter(),
}
}
if pathConfig.blockRequest != nil {
Expand Down
107 changes: 75 additions & 32 deletions tests/mitmproxy_addons/callback.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from urllib.error import HTTPError, URLError
from datetime import datetime

# Callback will intercept a response and send a POST request to the provided callback_url, with
# Callback will intercept a request and/or response and send a POST request to the provided url, with
# the following JSON object. Supports filters: https://docs.mitmproxy.org/stable/concepts-filters/
# {
# method: "GET|PUT|...",
Expand All @@ -20,10 +20,25 @@
# response_body: { some json object },
# response_code: 200,
# }
# Currently this is a read-only callback. The response cannot be modified, but side-effects can be
# taken. For example, tests may wish to terminate a client prior to the delivery of a response but
# after the server has processed the request, or the test may wish to use the response as a
# synchronisation point for a Waiter.
# The response to this request can control what gets returned to the client. The response object:
# {
# "respond_status_code": 200,
# "respond_body": { "some": "json_object" }
# }
# If {} is sent back, the response is not modified. Likewise, if `respond_body` is set but
# `respond_status_code` is not, only the response body is modified, not the status code, and vice versa.
Comment on lines +23 to +29
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This only applies to callback_response_url, right?

I think it might be clearer to document the request and response for the two URLs completely separately, even if there's a bit of duplication.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes.

Sounds good, I'm going to document this in the README where it's more easily accessible. But that will be in the next PR.

#
# To use this addon, configure it with these fields:
# - callback_request_url: the URL to send outbound requests to. This allows callbacks to intercept
# requests BEFORE they reach the server. The request/response struct in this
# callback is the same as `callback_response_url`, except `response_body`
# and `response_code` will be missing as the request hasn't been processed
# yet. To block the request from reaching the server, the callback needs to
# provide both `respond_status_code` and `respond_body`.
# - callback_response_url: the URL to send inbound responses to. This allows callbacks to modify
# response content.
# - filter: the mitmproxy filter to apply. If unset, ALL requests are eligible to go to the callback
# server.
class Callback:
def __init__(self):
self.reset()
Expand All @@ -32,38 +47,64 @@ def __init__(self):

def reset(self):
self.config = {
"callback_url": "",
"callback_request_url": "",
"callback_response_url": "",
Comment on lines +50 to +51
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

given that the whole thing is a Callback, the callback_ prefix for these names seems a bit redundant? YMMV

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I prefix this on purpose because there are a lot of request/response keys flying around (the ones you register with the addon, the ones passed in as req body to the callback server, the res body from the callback server) and I've tried to use callback_ for configuring the callback server, respond_ for the response body from the callback server and no special prefix for the callback data itself.

"filter": None,
}

def load(self, loader):
loader.add_option(
name="callback",
typespec=dict,
default={"callback_url": "", "filter": None},
default={
"callback_request_url": "",
"callback_response_url": "",
Comment on lines +60 to +61
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

seems like the options are, well, optional (we use config.get() rather than config[]) -- so maybe just omit these?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

They are optional but in absence of having __init__ set all the properties, I've tried to set them all in load, so readers can see what properties the addon uses. They aren't strictly required though as you say.

"filter": None,
},
help="Change the callback url, with an optional filter",
)

def configure(self, updates):
if "callback" not in updates:
self.reset()
return
if ctx.options.callback is None or ctx.options.callback["callback_url"] == "":
if ctx.options.callback is None:
self.reset()
return
self.config = ctx.options.callback
new_filter = self.config.get('filter', None)
print(f"callback will hit {self.config['callback_url']} filter={new_filter}")
print(f"callback req_url={self.config.get('callback_request_url')} res_url={self.config.get('callback_response_url')} filter={new_filter}")
if new_filter:
self.filter = flowfilter.parse(new_filter)
else:
self.filter = self.matchall

async def request(self, flow):
# always ignore the controller
if flow.request.pretty_host == MITM_DOMAIN_NAME:
return
if self.config.get("callback_request_url", "") == "":
return # ignore requests if we aren't told a url
if not flowfilter.match(self.filter, flow):
return # ignore requests which don't match the filter
try: # e.g GET requests have no req body
req_body = flow.request.json()
except:
req_body = None
print(f'{datetime.now().strftime("%H:%M:%S.%f")} hitting request callback for {flow.request.url}')
callback_body = {
"method": flow.request.method,
"access_token": flow.request.headers.get("Authorization", "").removeprefix("Bearer "),
"url": flow.request.url,
"request_body": req_body,
}
await self.send_callback(flow, self.config["callback_request_url"], callback_body)

async def response(self, flow):
# always ignore the controller
if flow.request.pretty_host == MITM_DOMAIN_NAME:
return
if self.config["callback_url"] == "":
if self.config.get("callback_response_url","") == "":
return # ignore responses if we aren't told a url
if flowfilter.match(self.filter, flow):
try: # e.g GET requests have no req body
Expand All @@ -74,7 +115,7 @@ async def response(self, flow):
res_body = flow.response.json()
except:
res_body = None
print(f'{datetime.now().strftime("%H:%M:%S.%f")} hitting callback for {flow.request.url}')
print(f'{datetime.now().strftime("%H:%M:%S.%f")} hitting response callback for {flow.request.url}')
callback_body = {
"method": flow.request.method,
"access_token": flow.request.headers.get("Authorization", "").removeprefix("Bearer "),
Expand All @@ -83,25 +124,27 @@ async def response(self, flow):
"request_body": req_body,
"response_body": res_body,
}
try:
# use asyncio so we don't block other unrelated requests from being processed
async with aiohttp.request(
method="POST",url=self.config["callback_url"], timeout=aiohttp.ClientTimeout(total=10),
headers={"Content-Type": "application/json"},
json=callback_body) as response:
print(f'{datetime.now().strftime("%H:%M:%S.%f")} callback for {flow.request.url} returned HTTP {response.status}')
test_response_body = await response.json()
# if the response includes some keys then we are modifying the response on a per-key basis.
if len(test_response_body) > 0:
respond_status_code = test_response_body.get("respond_status_code", flow.response.status_code)
respond_body = test_response_body.get("respond_body", res_body)
flow.response = Response.make(
respond_status_code, json.dumps(respond_body),
headers={
"MITM-Proxy": "yes", # so we don't reprocess this
"Content-Type": "application/json",
})
await self.send_callback(flow, self.config["callback_response_url"], callback_body)

except Exception as error:
print(f"ERR: callback for {flow.request.url} returned {error}")
print(f"ERR: callback, provided request body was {callback_body}")
async def send_callback(self, flow, url: str, body: dict):
try:
# use asyncio so we don't block other unrelated requests from being processed
async with aiohttp.request(
method="POST",url=url, timeout=aiohttp.ClientTimeout(total=10),
headers={"Content-Type": "application/json"},
json=body) as response:
kegsay marked this conversation as resolved.
Show resolved Hide resolved
print(f'{datetime.now().strftime("%H:%M:%S.%f")} callback for {flow.request.url} returned HTTP {response.status}')
test_response_body = await response.json()
# if the response includes some keys then we are modifying the response on a per-key basis.
if len(test_response_body) > 0:
respond_status_code = test_response_body.get("respond_status_code", flow.response.status_code)
respond_body = test_response_body.get("respond_body", body["response_body"])
flow.response = Response.make(
respond_status_code, json.dumps(respond_body),
headers={
"MITM-Proxy": "yes", # so we don't reprocess this
"Content-Type": "application/json",
})
except Exception as error:
print(f"ERR: callback for {flow.request.url} returned {error}")
print(f"ERR: callback, provided request body was {body}")
Loading