diff --git a/.circleci/config.yml b/.circleci/config.yml index 86f2599..4d639a7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,14 +2,15 @@ version: 2 jobs: build: docker: - - image: 218546966473.dkr.ecr.us-east-1.amazonaws.com/circle-ci:tap-tester-v4 + - image: 218546966473.dkr.ecr.us-east-1.amazonaws.com/circle-ci:stitch-tap-tester steps: - checkout - run: name: 'Setup' command: | - virtualenv -p python3 ~/.virtualenvs/tap-klaviyo - source ~/.virtualenvs/tap-klaviyo/bin/activate + virtualenv -p python3 /usr/local/share/virtualenvs/tap-klaviyo + source /usr/local/share/virtualenvs/tap-klaviyo/bin/activate + pip install -U pip 'setuptools<51.0.0' pip install . pip install pylint - run: @@ -20,15 +21,14 @@ jobs: - run: name: 'Pylint' command: | - source ~/.virtualenvs/tap-klaviyo/bin/activate - pylint tap_klaviyo -d C,R,'unspecified-encoding' + source /usr/local/share/virtualenvs/tap-klaviyo/bin/activate + pylint tap_klaviyo -d C,R,W - run: name: 'Unit Tests' command: | - source ~/.virtualenvs/tap-klaviyo/bin/activate - pip install nose coverage - nosetests --with-coverage --cover-erase --cover-package=tap_klaviyo --cover-html-dir=htmlcov tests/unittests - coverage html + source /usr/local/share/virtualenvs/tap-klaviyo/bin/activate + pip install nose2 parameterized nose2[coverage_plugin]>=0.6.5 + nose2 --with-coverage -v -s tests/unittests - store_test_results: path: test_output/report.xml - store_artifacts: @@ -40,14 +40,7 @@ jobs: aws s3 cp s3://com-stitchdata-dev-deployment-assets/environments/tap-tester/tap_tester_sandbox dev_env.sh source dev_env.sh source /usr/local/share/virtualenvs/tap-tester/bin/activate - run-test --tap=tap-klaviyo \ - --target=target-stitch \ - --orchestrator=stitch-orchestrator \ - --email=harrison+sandboxtest@stitchdata.com \ - --password=$SANDBOX_PASSWORD \ - --client-id=50 \ - --token=$STITCH_API_TOKEN \ - tests + run-test --tap=tap-klaviyo tests workflows: version: 2 diff --git a/README.md b/README.md index e8a23b2..9b3fc03 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ data from the Klaviyo API following the [Singer spec](https://github.com/singer-io/getting-started/blob/master/SPEC.md). This tap: -- Pulls raw data from the [Klaviyo metrics API](https://www.klaviyo.com/docs/api/metrics) +- Pulls raw data from the [Klaviyo metrics API](https://developers.klaviyo.com/en/reference/api_overview) - Outputs the schema for each resource - Incrementally pulls data based on the input state for incremental endpoints - Updates full tables for global exclusions and lists endpoints diff --git a/setup.py b/setup.py index 0996dd6..be4b1c1 100644 --- a/setup.py +++ b/setup.py @@ -9,8 +9,8 @@ url='http://singer.io', classifiers=['Programming Language :: Python :: 3 :: Only'], py_modules=['tap_klaviyo'], - install_requires=['singer-python==5.8.1', - 'requests==2.20.0'], + install_requires=['singer-python==6.0.0', + 'requests==2.31.0'], entry_points=''' [console_scripts] tap-klaviyo=tap_klaviyo:main diff --git a/tap_klaviyo/__init__.py b/tap_klaviyo/__init__.py index b8cd773..c0d5161 100644 --- a/tap_klaviyo/__init__.py +++ b/tap_klaviyo/__init__.py @@ -5,16 +5,19 @@ import sys import singer from singer import metadata -from tap_klaviyo.utils import get_incremental_pull, get_full_pulls, get_all_pages +from tap_klaviyo.utils import get_incremental_pull, get_full_pulls, get_all_using_next LOGGER = singer.get_logger() +API_VERSION = "2024-02-15" + +# For stream global_exclusions, data related to suppressed users can be found in the /api/profiles endpoint ENDPOINTS = { - 'global_exclusions': 'https://a.klaviyo.com/api/v1/people/exclusions', - 'lists': 'https://a.klaviyo.com/api/v1/lists', - 'metrics': 'https://a.klaviyo.com/api/v1/metrics', - 'metric': 'https://a.klaviyo.com/api/v1/metric/', - 'campaigns': 'https://a.klaviyo.com/api/v1/campaigns' + 'global_exclusions': 'https://a.klaviyo.com/api/profiles', + 'lists': 'https://a.klaviyo.com/api/lists', + 'metrics': 'https://a.klaviyo.com/api/metrics', + 'events': 'https://a.klaviyo.com/api/events', + 'campaigns': 'https://a.klaviyo.com/api/campaigns' } EVENT_MAPPINGS = { @@ -22,20 +25,21 @@ "Clicked Email": "click", "Opened Email": "open", "Bounced Email": "bounce", - "Unsubscribed": "unsubscribe", + "Unsubscribed from Email Marketing": "unsubscribe", "Marked Email as Spam": "mark_as_spam", "Unsubscribed from List": "unsub_list", + "Subscribed to Email Marketing": "subscribed_to_email", "Subscribed to List": "subscribe_list", "Updated Email Preferences": "update_email_preferences", "Dropped Email": "dropped_email", "Clicked SMS": "clicked_sms", - "Consented to Receive SMS": "consented_to_receive", + "Subscribed to SMS Marketing": "subscribed_to_sms", "Failed to Deliver SMS": "failed_to_deliver", "Failed to deliver Automated Response SMS": "failed_to_deliver_automated_response", "Received Automated Response SMS": "received_automated_response", "Received SMS": "received_sms", "Sent SMS": "sent_sms", - "Unsubscribed from SMS": "unsubscribed_from_sms" + "Unsubscribed from SMS Marketing": "unsubscribed_from_sms" } @@ -83,7 +87,7 @@ def to_catalog_dict(self): GLOBAL_EXCLUSIONS = Stream( 'global_exclusions', 'global_exclusions', - 'email', + 'id', 'FULL_TABLE' ) @@ -126,8 +130,7 @@ def load_shared_schema_refs(): return shared_schema_refs -def do_sync(config, state, catalog): - api_key = config['api_key'] +def do_sync(config, state, catalog, headers): start_date = config['start_date'] if 'start_date' in config else None stream_ids_to_sync = set() @@ -147,21 +150,21 @@ def do_sync(config, state, catalog): ) if stream['stream'] in EVENT_MAPPINGS.values(): - get_incremental_pull(stream, ENDPOINTS['metric'], state, - api_key, start_date) + get_incremental_pull(stream, ENDPOINTS['events'], state, + headers, start_date) else: - get_full_pulls(stream, ENDPOINTS[stream['stream']], api_key) + get_full_pulls(stream, ENDPOINTS[stream['stream']], headers) -def get_available_metrics(api_key): +def get_available_metrics(headers): metric_streams = [] - for response in get_all_pages('metric_list', - ENDPOINTS['metrics'], api_key): + for response in get_all_using_next('metric_list', + ENDPOINTS['metrics'], headers, {}): for metric in response.json().get('data'): - if metric['name'] in EVENT_MAPPINGS: + if metric['attributes']['name'] in EVENT_MAPPINGS: metric_streams.append( Stream( - stream=EVENT_MAPPINGS[metric['name']], + stream=EVENT_MAPPINGS[metric['attributes']['name']], tap_stream_id=metric['id'], key_properties="id", replication_method='INCREMENTAL', @@ -172,29 +175,32 @@ def get_available_metrics(api_key): return metric_streams -def discover(api_key): - metric_streams = get_available_metrics(api_key) +def discover(headers): + metric_streams = get_available_metrics(headers) return {"streams": [a.to_catalog_dict() for a in metric_streams + FULL_STREAMS]} -def do_discover(api_key): - print(json.dumps(discover(api_key), indent=2)) +def do_discover(headers): + print(json.dumps(discover(headers), indent=2)) @singer.utils.handle_top_exception(LOGGER) def main(): args = singer.utils.parse_args(REQUIRED_CONFIG_KEYS) + headers = { + "Authorization": f"Klaviyo-API-Key {args.config.get('api_key')}", + "revision": API_VERSION + } if args.discover: - do_discover(args.config['api_key']) + do_discover(headers) else: - catalog = args.catalog.to_dict() if args.catalog else discover( - args.config['api_key']) + catalog = args.catalog.to_dict() if args.catalog else discover(headers) state = args.state if args.state else {"bookmarks": {}} - do_sync(args.config, state, catalog) + do_sync(args.config, state, catalog, headers) if __name__ == '__main__': main() diff --git a/tap_klaviyo/schemas/campaigns.json b/tap_klaviyo/schemas/campaigns.json index 5a06835..e342199 100644 --- a/tap_klaviyo/schemas/campaigns.json +++ b/tap_klaviyo/schemas/campaigns.json @@ -10,145 +10,221 @@ "string" ] }, - "updated": { + "updated_at": { "type": [ "null", "string" ], "format": "date-time" }, - "from_name": { + "type": { "type": [ "null", "string" ] }, - "list_id": { + "send_time": { "type": [ "null", "string" - ] - }, - "template": { - "type": [ - "null", - "object" ], - "properties": { - "object": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "string" - ] - }, - "html": { - "type": [ - "null", - "string" - ] - } - } - }, - "status_id": { - "type": [ - "null", - "integer" - ] + "format": "date-time" }, - "object": { + "id": { "type": [ "null", "string" ] }, - "num_recipients": { + "name": { "type": [ "null", - "integer" + "string" ] }, - "lists": {}, - "is_segmented": { + "archived": { "type": [ "null", "boolean" ] }, - "send_time": { + "audiences": { "type": [ "null", - "string" + "object" ], - "format": "date-time" - }, - "excluded_lists": {}, - "id": { - "type": [ - "null", - "string" - ] + "properties": { + "included": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "string" + ] + } + }, + "excluded": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "string" + ] + } + } + } }, - "sent_at": { + "send_options": { "type": [ "null", - "string" + "object" ], - "format": "date-time" - }, - "campaign_type": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": [ - "null", - "string" - ] + "properties": {} }, - "created": { + "tracking_options": { "type": [ "null", - "string" + "object" ], - "format": "date-time" + "properties": {} }, - "status_label": { + "send_strategy": { "type": [ "null", - "string" - ] + "object" + ], + "properties": {} }, - "from_email": { + "campaign-messages": { "type": [ "null", - "string" - ] + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "label": { + "type": [ + "null", + "string" + ] + }, + "channel": { + "type": [ + "null", + "string" + ] + }, + "content": { + "type": [ + "null", + "object" + ], + "properties": {} + }, + "send_times": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": {} + } + }, + "render_options": { + "type": [ + "null", + "object" + ], + "properties": {} + }, + "created_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "updated_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + } + } + } }, - "subject": { + "tags": { "type": [ "null", - "string" - ] + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + } + } + } }, - "message_type": { + "scheduled_at": { "type": [ "null", "string" - ] + ], + "format": "date-time" }, - "template_id": { + "created_at": { "type": [ "null", "string" - ] + ], + "format": "date-time" } } } \ No newline at end of file diff --git a/tap_klaviyo/schemas/global_exclusions.json b/tap_klaviyo/schemas/global_exclusions.json index 3b9a3cb..f1eee34 100644 --- a/tap_klaviyo/schemas/global_exclusions.json +++ b/tap_klaviyo/schemas/global_exclusions.json @@ -4,24 +4,109 @@ "object" ], "properties": { - "timestamp": { + "type": { "type": [ "null", "string" ] }, - "reason": { + "id": { "type": [ "null", "string" ] }, - "object": { + "phone_number": { "type": [ "null", "string" ] }, + "external_id": { + "type": [ + "null", + "string" + ] + }, + "first_name": { + "type": [ + "null", + "string" + ] + }, + "last_name": { + "type": [ + "null", + "string" + ] + }, + "organization": { + "type": [ + "null", + "string" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "image": { + "type": [ + "null", + "string" + ] + }, + "created": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "updated": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "last_event_date": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "location": { + "type": [ + "null", + "object" + ], + "properties": {} + }, + "properties": { + "type": [ + "null", + "object" + ], + "properties": {} + }, + "subscriptions": { + "type": [ + "null", + "object" + ], + "properties": {} + }, + "predictive_analytics": { + "type": [ + "null", + "object" + ], + "properties": {} + }, "email": { "type": [ "null", @@ -29,4 +114,4 @@ ] } } -} +} \ No newline at end of file diff --git a/tap_klaviyo/schemas/lists.json b/tap_klaviyo/schemas/lists.json index b4cbc49..a3161a8 100644 --- a/tap_klaviyo/schemas/lists.json +++ b/tap_klaviyo/schemas/lists.json @@ -8,21 +8,17 @@ "type": [ "null", "string" - ] + ], + "format": "date-time" }, "updated": { - "type": [ - "null", - "string" - ] - }, - "person_count": { "type": [ "null", - "integer" - ] + "string" + ], + "format": "date-time" }, - "object": { + "type": { "type": [ "null", "string" @@ -34,11 +30,37 @@ "string" ] }, - "list_type": { + "tags": { "type": [ "null", - "string" - ] + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + } + } + } }, "id": { "type": [ @@ -46,11 +68,11 @@ "string" ] }, - "folder": { + "opt_in_process": { "type": [ "null", "string" ] } } -} +} \ No newline at end of file diff --git a/tap_klaviyo/schemas/shared/event.json b/tap_klaviyo/schemas/shared/event.json index 8775081..0ce932e 100644 --- a/tap_klaviyo/schemas/shared/event.json +++ b/tap_klaviyo/schemas/shared/event.json @@ -1,65 +1,207 @@ { - "type": [ - "null", - "object" - ], - "properties": { - "uuid": { - "type": [ - "null", - "string" - ] - }, - "event_name": { - "type": [ - "null", - "string" - ] - }, - "timestamp": { - "type": [ - "null", - "integer" - ] - }, - "object": { - "type": [ - "null", - "string" - ] - }, - "datetime": { - "type": [ - "null", - "string" - ] - }, - "statistic_id": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "string" - ] - }, - "event_properties": { + "type": [ + "null", + "object" + ], + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "uuid": { + "type": [ + "null", + "string" + ] + }, + "timestamp": { + "type": [ + "null", + "integer" + ] + }, + "datetime": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "event_properties": { + "type": [ + "null", + "object" + ], + "properties": {} + }, + "profile": { + "type": [ + "null", + "array" + ], + "items": { "type": [ "null", "object" ], - "properties": {} - }, - "person": { + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "email": { + "type": [ + "null", + "string" + ] + }, + "phone_number": { + "type": [ + "null", + "string" + ] + }, + "external_id": { + "type": [ + "null", + "string" + ] + }, + "first_name": { + "type": [ + "null", + "string" + ] + }, + "last_name": { + "type": [ + "null", + "string" + ] + }, + "organization": { + "type": [ + "null", + "string" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "image": { + "type": [ + "null", + "string" + ] + }, + "created": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "updated": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "last_event_date": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "location": { + "type": [ + "null", + "object" + ], + "properties": {} + }, + "properties": { + "type": [ + "null", + "object" + ], + "properties": {} + } + } + } + }, + "metric": { + "type": [ + "null", + "array" + ], + "items": { "type": [ "null", "object" ], - "properties": {} + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "created": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "updated": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "integration": { + "type": [ + "null", + "object" + ], + "properties": {} + } + } } } } - \ No newline at end of file +} \ No newline at end of file diff --git a/tap_klaviyo/schemas/consented_to_receive.json b/tap_klaviyo/schemas/subscribed_to_email.json similarity index 100% rename from tap_klaviyo/schemas/consented_to_receive.json rename to tap_klaviyo/schemas/subscribed_to_email.json diff --git a/tap_klaviyo/schemas/subscribed_to_sms.json b/tap_klaviyo/schemas/subscribed_to_sms.json new file mode 100644 index 0000000..443199a --- /dev/null +++ b/tap_klaviyo/schemas/subscribed_to_sms.json @@ -0,0 +1 @@ +{"$ref": "shared/event.json#/"} \ No newline at end of file diff --git a/tap_klaviyo/utils.py b/tap_klaviyo/utils.py index 9c27c6b..ec2e265 100644 --- a/tap_klaviyo/utils.py +++ b/tap_klaviyo/utils.py @@ -14,6 +14,39 @@ session = requests.Session() logger = singer.get_logger() +STREAM_PARAMS_MAP = { + "campaigns": [ + { + "filter": "equals(messages.channel,'email')", + "include": "tags,campaign-messages" + } + ], + "global_exclusions": [ + { + "filter": "equals(subscriptions.email.marketing.suppression.reason,'HARD_BOUNCE')", + "additional-fields[profile]": "subscriptions,predictive_analytics" + }, + { + "filter": "equals(subscriptions.email.marketing.suppression.reason,'USER_SUPPRESSED')", + "additional-fields[profile]": "subscriptions,predictive_analytics" + }, + { + "filter": "equals(subscriptions.email.marketing.suppression.reason,'UNSUBSCRIBE')", + "additional-fields[profile]": "subscriptions,predictive_analytics" + }, + { + "filter": "equals(subscriptions.email.marketing.suppression.reason,'INVALID_EMAIL')", + "additional-fields[profile]": "subscriptions,predictive_analytics" + } + ], + "lists": [ + { + "include": "tags" + } + ] + +} + class KlaviyoError(Exception): pass @@ -168,9 +201,9 @@ def get_latest_event_time(events): # hence added backoff for 'ConnectionError' too. @backoff.on_exception(backoff.expo, (requests.Timeout, requests.ConnectionError), max_tries=5, factor=2) @backoff.on_exception(backoff.expo, (simplejson.scanner.JSONDecodeError, KlaviyoBackoffError), max_tries=3) -def authed_get(source, url, params): +def authed_get(source, url, params, headers): with metrics.http_request_timer(source) as timer: - resp = session.request(method='get', url=url, params=params, timeout=get_request_timeout()) + resp = requests.get(url=url, params=params, headers=headers, timeout=get_request_timeout()) if resp.status_code != 200: raise_for_error(resp) @@ -179,68 +212,82 @@ def authed_get(source, url, params): timer.tags[metrics.Tag.http_status_code] = resp.status_code return resp - -def get_all_using_next(stream, url, api_key, since=None): - while True: - r = authed_get(stream, url, {'api_key': api_key, - 'since': since, - 'sort': 'asc'}) - yield r - if 'next' in r.json() and r.json()['next']: - since = r.json()['next'] - else: - break - - -def get_all_pages(source, url, api_key): - page = 0 - while True: - r = authed_get(source, url, {'page': page, 'api_key': api_key}) +def get_all_using_next(stream, url, headers, params): + # Paginate till there is a url or next url. + while url: + r = authed_get(stream, url, params, headers) + # Re-initializing params to {} as next url contains all necessary params. + params = {} yield r - if r.json()['end'] < r.json()['total'] - 1: - page += 1 - else: - break + url = r.json()['links'].get('next', None) -def get_incremental_pull(stream, endpoint, state, api_key, start_date): +def get_incremental_pull(stream, endpoint, state, headers, start_date): latest_event_time = get_starting_point(stream, state, start_date) with metrics.record_counter(stream['stream']) as counter: - url = '{}{}/timeline'.format( - endpoint, - stream['tap_stream_id'] - ) - for response in get_all_using_next( - stream['stream'], url, api_key, - latest_event_time): + params = { + "filter": f"equals(metric_id,\"{stream['tap_stream_id']}\"),greater-or-equal(timestamp,{latest_event_time})", + "include": "profile,metric", + "sort": "datetime" + } + for response in get_all_using_next(stream['stream'], endpoint, headers, params): events = response.json().get('data') if events: + included_list = response.json().get('included', []) + # Creating a dict/map of included relationships to optimize computations + included = {} + for included_relationship in included_list: + included[included_relationship['id']] = included_relationship counter.increment(len(events)) - transfrom_and_write_records(events, stream) + transfrom_and_write_records(events, stream, included, params.get("include","").split(",")) update_state(state, stream['stream'], get_latest_event_time(events)) singer.write_state(state) return state -def get_full_pulls(resource, endpoint, api_key): +def get_full_pulls(resource, endpoint, headers): with metrics.record_counter(resource['stream']) as counter: - - for response in get_all_pages(resource['stream'], endpoint, api_key): - - records = response.json().get('data') - counter.increment(len(records)) - transfrom_and_write_records(records, resource) - - -def transfrom_and_write_records(events, stream): + for params in STREAM_PARAMS_MAP.get(resource['stream'],[]): + for response in get_all_using_next(resource['stream'], endpoint, headers, params): + records = response.json().get('data') + included_list = response.json().get('included', []) + # Creating a dict/map of included relationships to optimize computations + included = {} + for included_relationship in included_list: + included[included_relationship['id']] = included_relationship + counter.increment(len(records)) + transfrom_and_write_records(records, resource, included, params.get("include","").split(",")) + + +def transfrom_and_write_records(events, stream, included, valid_relationships): event_stream = stream['stream'] event_schema = stream['schema'] event_mdata = metadata.to_map(stream['metadata']) with Transformer() as transformer: for event in events: + # Flatten the event dict with attributes + event.update(event['attributes']) + for relationship_key, relationship_value in event.get('relationships',{}).items(): + if not relationship_key in valid_relationships: + continue + relationship_data = relationship_value['data'] + # Generalizing relationship data to list of dicts for all streams + # This is due to the fact that, for Full table streams, data is returned as a list + # And, for incremental streams, data is return as a dict in API response + if isinstance(relationship_data, dict): + relationship_data = [relationship_data] + for relationship in relationship_data: + included_relationship = included.get(relationship['id'], None) + # Check if current relationship is present in included relationship dict + if included_relationship is not None: + # Flatten the included_relationship dict with attributes + included_relationship.update(included_relationship['attributes']) + relationship.update(included_relationship) + event.update({relationship_key: relationship_data}) + # write record singer.write_record( event_stream, transformer.transform( diff --git a/tests/base.py b/tests/base.py index 9caf686..433f8a2 100644 --- a/tests/base.py +++ b/tests/base.py @@ -119,6 +119,18 @@ def expected_metadata(self): self.REPLICATION_KEYS:{"since"}, self.BOOKMARK: {"timestamp"} }, + "subscribed_to_email": { + self.PRIMARY_KEYS: {"id"}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS:{"since"}, + self.BOOKMARK: {"timestamp"} + }, + "subscribed_to_sms": { + self.PRIMARY_KEYS: {"id"}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS:{"since"}, + self.BOOKMARK: {"timestamp"} + }, "update_email_preferences": { self.PRIMARY_KEYS: {"id"}, self.REPLICATION_METHOD: self.INCREMENTAL, @@ -126,7 +138,7 @@ def expected_metadata(self): self.BOOKMARK: {"timestamp"} }, "global_exclusions": { - self.PRIMARY_KEYS: {"email"}, + self.PRIMARY_KEYS: {"id"}, self.REPLICATION_METHOD: self.FULL_TABLE }, "lists": { @@ -149,12 +161,6 @@ def expected_metadata(self): self.REPLICATION_KEYS:{"since"}, self.BOOKMARK: {"timestamp"} }, - 'consented_to_receive': { - self.PRIMARY_KEYS: {"id"}, - self.REPLICATION_METHOD: self.INCREMENTAL, - self.REPLICATION_KEYS:{"since"}, - self.BOOKMARK: {"timestamp"} - }, 'received_sms': { self.PRIMARY_KEYS: {"id"}, self.REPLICATION_METHOD: self.INCREMENTAL, @@ -244,7 +250,9 @@ def run_and_verify_check_mode(self, conn_id): found_catalog_names = set(map(lambda c: c['stream_name'], found_catalogs)) print(found_catalog_names) - self.assertSetEqual(self.expected_streams(), found_catalog_names, msg="discovered schemas do not match") + # In klaviyo, on the metrics api endpoint data, we finalize the catalog + # And, it is not necessary that all schemas defined in /schemas will be part of catalog. + self.assertTrue(set(found_catalog_names).issubset(set(self.expected_streams())), msg="discovered schemas do not match") print("discovered schemas are OK") return found_catalogs diff --git a/tests/test_klaviyo_all_fields.py b/tests/test_klaviyo_all_fields.py index bd1d332..a0f2663 100644 --- a/tests/test_klaviyo_all_fields.py +++ b/tests/test_klaviyo_all_fields.py @@ -19,8 +19,9 @@ def test_all_fields_run(self): """ - # Streams to verify all fields tests - expected_streams = self.expected_streams() - {"mark_as_spam", "dropped_email"} + # Test account does not have data for untestable streams + untestable_streams = {"unsubscribe", "mark_as_spam", "dropped_email"} + expected_streams = self.expected_streams() - untestable_streams expected_automatic_fields = self.expected_automatic_fields() conn_id = connections.ensure_connection(self) diff --git a/tests/test_klaviyo_automatic_fields.py b/tests/test_klaviyo_automatic_fields.py index 8db5bef..8d86f28 100644 --- a/tests/test_klaviyo_automatic_fields.py +++ b/tests/test_klaviyo_automatic_fields.py @@ -18,8 +18,10 @@ def test_run(self): • Verify that only the automatic fields are sent to the target. • Verify that all replicated records have unique primary key values. """ - # We are not able to generate test data so skipping two streams(mark_as_spam, dropped_email) - expected_streams = self.expected_streams() - {"mark_as_spam", "dropped_email"} + + # Test account does not have data for untestable streams + untestable_streams = {"unsubscribe", "mark_as_spam", "dropped_email"} + expected_streams = self.expected_streams() - untestable_streams conn_id = connections.ensure_connection(self) diff --git a/tests/test_klaviyo_bookmark.py b/tests/test_klaviyo_bookmark.py index 454892d..5da3a75 100644 --- a/tests/test_klaviyo_bookmark.py +++ b/tests/test_klaviyo_bookmark.py @@ -28,8 +28,9 @@ def test_run(self): different values for the replication key """ - # We are not able to generate test data so skipping two streams(mark_as_spam, dropped_email) - expected_streams = self.expected_streams() - {"mark_as_spam", "dropped_email"} + # Test account does not have data for untestable streams + untestable_streams = {"unsubscribe", "mark_as_spam", "dropped_email"} + expected_streams = self.expected_streams() - untestable_streams expected_bookmark_keys = self.expected_bookmark_keys() expected_replication_keys = self.expected_replication_keys() diff --git a/tests/test_klaviyo_discovery.py b/tests/test_klaviyo_discovery.py index 30a2271..8268c43 100644 --- a/tests/test_klaviyo_discovery.py +++ b/tests/test_klaviyo_discovery.py @@ -29,7 +29,9 @@ def test_run(self): • verify replication key(s) • verify that if there is a replication key we are doing INCREMENTAL otherwise FULL """ - streams_to_test = self.expected_streams() + # Test account does not have data for untestable streams + untestable_streams = {"unsubscribe"} + streams_to_test = self.expected_streams() - untestable_streams conn_id = connections.ensure_connection(self) diff --git a/tests/test_klaviyo_pagination.py b/tests/test_klaviyo_pagination.py index 7cc748f..6c41d17 100644 --- a/tests/test_klaviyo_pagination.py +++ b/tests/test_klaviyo_pagination.py @@ -22,9 +22,18 @@ def test_run(self): page_size = 100 # Page size for opened emails conn_id = connections.ensure_connection(self) - # Skipped the below streams as we are not able to generate enough records to pass pagination test cases. - # It shows the message as - You have reached 100% of your monthly sending limit - expected_streams = self.expected_streams() - {"mark_as_spam", "dropped_email","received_sms","failed_to_deliver","failed_to_deliver_automated_response"} + # Test account does not have data for untestable streams + untestable_streams = {"unsubscribe", "mark_as_spam", "dropped_email"} + expected_streams = self.expected_streams() - untestable_streams + stream_page_size = {} + for stream in expected_streams: + stream_page_size[stream] = page_size + # Page size for streams are set based on available data in test account + stream_page_size["received_sms"] = 50 + stream_page_size["failed_to_deliver_automated_response"] = 20 + stream_page_size["subscribed_to_email"] = 1 + stream_page_size["failed_to_deliver"] = 1 + found_catalogs = self.run_and_verify_check_mode(conn_id) # table and field selection @@ -49,7 +58,7 @@ def test_run(self): if message.get('action') == 'upsert'] # verify records are more than page size so multiple page is working - self.assertGreater(record_count_sync, page_size) + self.assertGreater(record_count_sync, stream_page_size[stream]) if record_count_sync > page_size: primary_keys_list_1 = primary_keys_list[:page_size] diff --git a/tests/test_klaviyo_start_date.py b/tests/test_klaviyo_start_date.py index cefe9fe..b0cb772 100644 --- a/tests/test_klaviyo_start_date.py +++ b/tests/test_klaviyo_start_date.py @@ -16,15 +16,14 @@ def name(): def test_run(self): - # We are not able to generate test data so skipping two streams(mark_as_spam, dropped_email) - # Skipping given streams due to data available only on a single date - failed_to_deliver_automated_response, failed_to_deliver + # Test account does not have data for untestable streams + untestable_streams = {"unsubscribe", "mark_as_spam", "dropped_email"} - expected_streams = self.expected_streams() - {"mark_as_spam", "dropped_email", "failed_to_deliver_automated_response", - "failed_to_deliver"} + expected_streams = self.expected_streams() - untestable_streams # running start_date_test for old streams - expected_streams_1 = {"receive", "click", "open", "bounce", "unsubscribe", "unsub_list", "subscribe_list", - "update_email_preferences","global_exclusions","lists","campaigns"} + expected_streams_1 = {"receive", "click", "open", "bounce", "unsub_list", "subscribe_list", + "update_email_preferences", "global_exclusions", "lists", "campaigns"} self.run_start_date(expected_streams_1, days = 3) # running start_date_test for newly added streams @@ -132,9 +131,15 @@ def run_start_date(self, expected_streams, days = 0): for bookmark_key_value in bookmark_key_sync_2: self.assertGreaterEqual(bookmark_key_value, start_date_2_epoch) - # Verify the number of records replicated in sync 1 is greater than the number - # of records replicated in sync 2 for stream - self.assertGreater(record_count_sync_1, record_count_sync_2) + if stream in ['failed_to_deliver_automated_response', 'failed_to_deliver', 'subscribed_to_email']: + # Verify the number of records replicated in sync 1 is greater than or equal to + # the number of records replicated in sync 2 for mentioned streams as + # the data is available for only 1 day for these streams + self.assertGreaterEqual(record_count_sync_1, record_count_sync_2) + else: + # Verify the number of records replicated in sync 1 is greater than the number + # of records replicated in sync 2 for stream + self.assertGreater(record_count_sync_1, record_count_sync_2) # Verify the records replicated in sync 2 were also replicated in sync 1 self.assertTrue(primary_keys_sync_2.issubset(primary_keys_sync_1)) diff --git a/tests/unittests/test_backoff.py b/tests/unittests/test_backoff.py index 3eaa111..370f7db 100644 --- a/tests/unittests/test_backoff.py +++ b/tests/unittests/test_backoff.py @@ -11,24 +11,24 @@ @mock.patch("tap_klaviyo.utils.get_request_timeout") @mock.patch('requests.Session.request') class TestBackoff(unittest.TestCase): - + def test_internal_service_error(self, mocked_session, mocked_get_request_timeout, mocked_sleep): """ Check whether the request backoffs properly for authed_get() for 3 times in case of KlaviyoInternalServiceError. """ mock_resp = mock.Mock() - + klaviyo_error = utils_.KlaviyoInternalServiceError() http_error = requests.HTTPError() mock_resp.raise_for_error.side_effect = klaviyo_error mock_resp.raise_for_status.side_effect = http_error mock_resp.status_code = 500 - + mocked_session.return_value = mock_resp try: - utils_.authed_get("", "", "") + utils_.authed_get("", "", "", "") except utils_.KlaviyoInternalServiceError: pass @@ -47,7 +47,7 @@ def test_jsondecode(self, mocked_request, mocked_get_request_timeout, mocked_sle mocked_request.return_value = mock_resp try: - utils_.authed_get("", "", "") + utils_.authed_get("", "", "", "") except simplejson.scanner.JSONDecodeError: pass @@ -59,21 +59,21 @@ def test_not_implemented_error(self, mocked_session, mocked_get_request_timeout, Check whether the request backoffs properly for authed_get() for 3 times in case of KlaviyoNotImplementedError. """ mock_resp = mock.Mock() - + klaviyo_error = utils_.KlaviyoNotImplementedError() http_error = requests.HTTPError() mock_resp.raise_for_error.side_effect = klaviyo_error mock_resp.raise_for_status.side_effect = http_error mock_resp.status_code = 501 - + mocked_session.return_value = mock_resp try: - utils_.authed_get("", "", "") + utils_.authed_get("", "", "", "") except utils_.KlaviyoNotImplementedError: pass - + # verify that we backoff for 3 times self.assertEquals(mocked_session.call_count, 3) @@ -82,18 +82,18 @@ def test_bad_gateway_error(self, mocked_session, mocked_get_request_timeout, moc Check whether the request backoffs properly for authed_get() for 3 times in case of KlaviyoBadGatewayError. """ mock_resp = mock.Mock() - + klaviyo_error = utils_.KlaviyoBadGatewayError() http_error = requests.HTTPError() mock_resp.raise_for_error.side_effect = klaviyo_error mock_resp.raise_for_status.side_effect = http_error mock_resp.status_code = 502 - + mocked_session.return_value = mock_resp try: - utils_.authed_get("", "", "") + utils_.authed_get("", "", "", "") except utils_.KlaviyoBadGatewayError: pass @@ -105,18 +105,18 @@ def test_service_unavailable_error(self, mocked_session, mocked_get_request_time Check whether the request backoffs properly for authed_get() for 3 times in case of KlaviyoServiceUnavailableError. """ mock_resp = mock.Mock() - + klaviyo_error = utils_.KlaviyoServiceUnavailableError() http_error = requests.HTTPError() mock_resp.raise_for_error.side_effect = klaviyo_error mock_resp.raise_for_status.side_effect = http_error mock_resp.status_code = 503 - + mocked_session.return_value = mock_resp try: - utils_.authed_get("", "", "") + utils_.authed_get("", "", "", "") except utils_.KlaviyoServiceUnavailableError: pass @@ -128,18 +128,18 @@ def test_gateway_timeout_error(self, mocked_session, mocked_get_request_timeout, Check whether the request backoffs properly for authed_get() for 3 times in case of KlaviyoGatewayTimeoutError. """ mock_resp = mock.Mock() - + klaviyo_error = utils_.KlaviyoGatewayTimeoutError() http_error = requests.HTTPError() mock_resp.raise_for_error.side_effect = klaviyo_error mock_resp.raise_for_status.side_effect = http_error mock_resp.status_code = 504 - + mocked_session.return_value = mock_resp try: - utils_.authed_get("", "", "") + utils_.authed_get("", "", "", "") except utils_.KlaviyoGatewayTimeoutError: pass @@ -151,42 +151,42 @@ def test_server_timeout_error(self, mocked_session, mocked_get_request_timeout, Check whether the request backoffs properly for authed_get() for 3 times in case of KlaviyoServerTimeoutError. """ mock_resp = mock.Mock() - + klaviyo_error = utils_.KlaviyoServerTimeoutError() http_error = requests.HTTPError() mock_resp.raise_for_error.side_effect = klaviyo_error mock_resp.raise_for_status.side_effect = http_error mock_resp.status_code = 524 - + mocked_session.return_value = mock_resp try: - utils_.authed_get("", "", "") + utils_.authed_get("", "", "", "") except utils_.KlaviyoServerTimeoutError: pass # verify that we backoff for 3 times self.assertEquals(mocked_session.call_count, 3) - + def test_rate_limit_error(self, mocked_session, mocked_get_request_timeout, mocked_sleep): """ Check whether the request backoffs properly for authed_get() for 3 times in case of KlaviyoRateLimitError. """ mock_resp = mock.Mock() - + klaviyo_error = utils_.KlaviyoRateLimitError() http_error = requests.HTTPError() mock_resp.raise_for_error.side_effect = klaviyo_error mock_resp.raise_for_status.side_effect = http_error mock_resp.status_code = 429 - + mocked_session.return_value = mock_resp try: - utils_.authed_get("", "", "") + utils_.authed_get("", "", "", "") except utils_.KlaviyoRateLimitError: pass diff --git a/tests/unittests/test_exception_handling.py b/tests/unittests/test_exception_handling.py index e515d29..325099b 100644 --- a/tests/unittests/test_exception_handling.py +++ b/tests/unittests/test_exception_handling.py @@ -2,8 +2,7 @@ import unittest from unittest import mock import requests -import singer -import json + class Mockresponse: def __init__(self, status_code, resp={}, content=None, headers=None, raise_error=False): @@ -93,16 +92,16 @@ class TestBackoff(unittest.TestCase): def test_200(self, successful_200_request, mocked_get_request_timeout): test_data = {"tap": "klaviyo", "code": 200} - actual_data = utils_.authed_get("", "", "").json() + actual_data = utils_.authed_get("", "", "", "").json() self.assertEquals(actual_data, test_data) - + @mock.patch('requests.Session.request', side_effect=klaviyo_400_error) def test_400_error(self, klaviyo_400_error, mocked_get_request_timeout): """ Test that `authed_get` raise 400 error with proper message """ try: - utils_.authed_get("", "", "") + utils_.authed_get("", "", "", "") except utils_.KlaviyoBadRequestError as e: self.assertEquals(str(e), "HTTP-error-code: 400, Error: Request is missing or has a bad parameter.") @@ -112,7 +111,7 @@ def test_401_error(self, klaviyo_401_error, mocked_get_request_timeout): Test that `authed_get` raise 401 error with proper message """ try: - utils_.authed_get("", "", "") + utils_.authed_get("", "", "", "") except utils_.KlaviyoUnauthorizedError as e: self.assertEquals(str(e), "HTTP-error-code: 401, Error: Invalid authorization credentials.") @@ -122,7 +121,7 @@ def test_403_error(self, klaviyo_403_error, mocked_get_request_timeout): Test that `authed_get` raise 403 error with proper message """ try: - utils_.authed_get("", "", "") + utils_.authed_get("", "", "", "") except utils_.KlaviyoForbiddenError as e: self.assertEquals(str(e), "HTTP-error-code: 403, Error: Invalid authorization credentials or permissions.") @@ -132,7 +131,7 @@ def test_403_error_wrong_api_key(self, klaviyo_403_error_wrong_api_key, mocked_g Test that `authed_get` raise 403 error with proper message for wrong api key """ try: - utils_.authed_get("", "", "") + utils_.authed_get("", "", "", "") except utils_.KlaviyoForbiddenError as e: self.assertEquals(str(e), "HTTP-error-code: 403, Error: The API key specified is invalid.") @@ -142,7 +141,7 @@ def test_403_error_missing_api_key(self, klaviyo_403_error_missing_api_key, mock Test that `authed_get` raise 403 error with proper message for empty api key """ try: - utils_.authed_get("", "", "") + utils_.authed_get("", "", "", "") except utils_.KlaviyoForbiddenError as e: self.assertEquals(str(e), "HTTP-error-code: 403, Error: You must specify an API key to make requests.") @@ -152,7 +151,7 @@ def test_404_error(self, klaviyo_404_error, mocked_get_request_timeout): Test that `authed_get` raise 404 error with proper message """ try: - utils_.authed_get("", "", "") + utils_.authed_get("", "", "", "") except utils_.KlaviyoNotFoundError as e: self.assertEquals(str(e), "HTTP-error-code: 404, Error: The requested resource doesn't exist.") @@ -162,7 +161,7 @@ def test_409_error(self, klaviyo_409_error, mocked_get_request_timeout): Test that `authed_get` raise 409 error with proper message """ try: - utils_.authed_get("", "", "") + utils_.authed_get("", "", "", "") except utils_.KlaviyoConflictError as e: self.assertEquals(str(e), "HTTP-error-code: 409, Error: The API request cannot be completed because the requested operation would conflict with an existing item.") @@ -173,7 +172,7 @@ def test_500_error(self, klaviyo_500_error, mocked_sleep, mocked_get_request_tim Test that `authed_get` raise 500 error with proper message """ try: - utils_.authed_get("", "", "") + utils_.authed_get("", "", "", "") except utils_.KlaviyoInternalServiceError as e: self.assertEquals(str(e), "HTTP-error-code: 500, Error: Internal Service Error from Klaviyo.") @@ -184,7 +183,7 @@ def test_429_error(self, klaviyo_429_error, mocked_sleep, mocked_get_request_tim Test that `authed_get` raise 429 error with proper message """ try: - utils_.authed_get("", "", "") + utils_.authed_get("", "", "", "") except utils_.KlaviyoRateLimitError as e: self.assertEquals(str(e), "HTTP-error-code: 429, Error: The API rate limit for your organization/application pairing has been exceeded.") @@ -195,7 +194,7 @@ def test_501_error(self, klaviyo_501_error, mocked_sleep, mocked_get_request_tim Test that `authed_get` raise 501 error with proper message """ try: - utils_.authed_get("", "", "") + utils_.authed_get("", "", "", "") except utils_.KlaviyoNotImplementedError as e: self.assertEquals(str(e), "HTTP-error-code: 501, Error: The server does not support the functionality required to fulfill the request.") @@ -206,7 +205,7 @@ def test_502_error(self, klaviyo_502_error, mocked_sleep, mocked_get_request_tim Test that `authed_get` raise 502 error with proper message """ try: - utils_.authed_get("", "", "") + utils_.authed_get("", "", "", "") except utils_.KlaviyoBadGatewayError as e: self.assertEquals(str(e), "HTTP-error-code: 502, Error: Server received an invalid response from another server.") @@ -217,7 +216,7 @@ def test_503_error(self, klaviyo_503_error, mocked_sleep, mocked_get_request_tim Test that `authed_get` raise 503 error with proper message """ try: - utils_.authed_get("", "", "") + utils_.authed_get("", "", "", "") except utils_.KlaviyoServiceUnavailableError as e: self.assertEquals(str(e), "HTTP-error-code: 503, Error: API service is currently unavailable.") @@ -228,7 +227,7 @@ def test_504_error(self, klaviyo_504_error, mocked_sleep, mocked_get_request_tim Test that `authed_get` raise 504 error with proper message """ try: - utils_.authed_get("", "", "") + utils_.authed_get("", "", "", "") except utils_.KlaviyoGatewayTimeoutError as e: self.assertEquals(str(e), "HTTP-error-code: 504, Error: Server did not return a response from another server.") @@ -239,6 +238,6 @@ def test_524_error(self, klaviyo_524_error, mocked_sleep, mocked_get_request_tim Test that `authed_get` raise 524 error with proper message """ try: - utils_.authed_get("", "", "") + utils_.authed_get("", "", "", "") except utils_.KlaviyoServerTimeoutError as e: self.assertEquals(str(e), "HTTP-error-code: 524, Error: Server took too long to respond to the request.") diff --git a/tests/unittests/test_fields_inclusion_in_metadata.py b/tests/unittests/test_fields_inclusion_in_metadata.py index 156ebef..e65ab18 100644 --- a/tests/unittests/test_fields_inclusion_in_metadata.py +++ b/tests/unittests/test_fields_inclusion_in_metadata.py @@ -17,28 +17,28 @@ class TestFieldsInclusionInMetadata(unittest.TestCase): """ Test cases to verify inclusion value is available for fields in metadata and automatic for key_property """ - - @mock.patch("tap_klaviyo.get_all_pages") - def test_fields_inclusion_in_metadata(self, mock_get_all_pages): - api_key = "abc" - streams = [{"name": "Received Email", "id": "UfdfRD"}, - {"name": "Clicked Email", "id": "UrdfRD"}, - {"name": "Opened Email", "id": "UfdfRu"}, - {"name": "Bounced Email", "id": "RfdfRD"}, - {"name": "Unsubscribed", "id": "UddfRD"}, - {"name": "Marked Email as Spam", "id": "UffgbD"}, - {"name": "Unsubscribed from List", "id": "UsfewD"}, - {"name": "Subscribed to List", "id": "UrDfwD"}, - {"name": "Updated Email Preferences", "id": "AdfDfD"}, - {"name": "Dropped Email", "id": "AfdfdD"}] + + @mock.patch("tap_klaviyo.get_all_using_next") + def test_fields_inclusion_in_metadata(self, mock_get_all_using_next): + header = "abc" + streams = [{"attributes": {"name": "Received Email"}, "id": "UfdfRD"}, + {"attributes": {"name": "Clicked Email"}, "id": "UrdfRD"}, + {"attributes": {"name": "Opened Email"}, "id": "UfdfRu"}, + {"attributes": {"name": "Bounced Email"}, "id": "RfdfRD"}, + {"attributes": {"name": "Unsubscribed"}, "id": "UddfRD"}, + {"attributes": {"name": "Marked Email as Spam"}, "id": "UffgbD"}, + {"attributes": {"name": "Unsubscribed from List"}, "id": "UsfewD"}, + {"attributes": {"name": "Subscribed to List"}, "id": "UrDfwD"}, + {"attributes": {"name": "Updated Email Preferences"}, "id": "AdfDfD"}, + {"attributes": {"name": "Dropped Email"}, "id": "AfdfdD"}] full_table_stream = ["global_exclusions", "lists", "campaigns"] bookmark_key = 'timestamp' - mock_get_all_pages.return_value = [get_mock_http_response( + mock_get_all_using_next.return_value = [get_mock_http_response( 200, {"data": streams})] # Get catalog - catalog = tap_klaviyo.discover(api_key) + catalog = tap_klaviyo.discover(header) catalog = catalog['streams'] for catalog_entry in catalog: diff --git a/tests/unittests/test_null_values.py b/tests/unittests/test_null_values.py index 3526768..9ced2f3 100644 --- a/tests/unittests/test_null_values.py +++ b/tests/unittests/test_null_values.py @@ -1,8 +1,6 @@ -import tap_klaviyo import unittest from unittest import mock import singer -from singer import metrics, metadata, Transformer from tap_klaviyo.utils import transfrom_and_write_records from tap_klaviyo import discover @@ -14,7 +12,7 @@ def __init__(self, status_code, resp={}, content=None, headers=None, raise_error self.content = content self.headers = headers self.raise_error = raise_error - + def json(self): return self.json_data @@ -22,7 +20,7 @@ def raise_for_status(self): raise requests.HTTPError def successful_200_request(*args, **kwargs): - json_str = {"data": [{'object': 'metric', 'id': 'dummy_id', 'name': 'Fulfilled Order', 'integration': {'object': 'integration', 'id': '0eMvjm', 'name': 'Shopify', 'category': 'eCommerce'}, 'created': '2021-09-15 09:54:46', 'updated': '2021-09-15 09:54:46'}], 'page': 0, 'start': 0, 'end': 28, 'total': 29, 'page_size': 29} + json_str = {"data": [{"type": "metric", "id": "abc", "attributes": {"name": "abc", "created": "2019-09-09T19:06:57+00:00", "updated": "2019-09-09T19:06:57+00:00", "integration": {"object": "integration", "id": "abc"}}}],"links": {"self": "abc", "next": None, "previous": None}} return MockResponse(200, json_str) @@ -44,9 +42,9 @@ def test_null_values(self, mock_200, mock_parse_args): config = {} mock_parse_args.return_value = MockParseArgs(state = {}, discover = True, config=config) # record with null values - records = [{'object': 'campaign', 'id': None, 'name': None, 'subject': None, 'from_email': None, 'from_name': None, 'lists': None, 'excluded_lists': [], 'status': None, 'status_id': None, 'status_label': None, 'sent_at': None, 'send_time': None, 'created': None, 'updated': None, 'num_recipients': None, 'campaign_type': None, 'is_segmented': None, 'message_type': None, 'template_id': None}] + records = [{'type': 'campaign', 'id': None, 'attributes': {'name': None, 'status': None, 'archived': None, 'audiences': {'included': [None], 'excluded': [None]}, 'send_options': {'use_smart_sending': None, 'ignore_unsubscribes': None}, 'tracking_options': {'is_add_utm': None, 'utm_params': [], 'is_tracking_clicks': None, 'is_tracking_opens': None}, 'send_strategy': {'method': None, 'options_static': None, 'options_throttled': None, 'options_sto': None}, 'created_at': None, 'scheduled_at': None, 'updated_at': None, 'send_time': None}, 'relationships': {'campaign-messages': {'data': [{'type': 'campaign-message', 'id': None}], 'links': {'self': None, 'related': None}}, 'tags': {'data': [], 'links': {'self': None, 'related': None}}}, 'links': {'self': None}}] catalog = discover("dummy_key") stream = None stream = [each for each in catalog['streams'] if each['stream'] == 'campaigns'] # Verify by calling `transfrom_and_write_records` that it doesn't throw any exception while transform - transfrom_and_write_records(records, stream[0]) \ No newline at end of file + transfrom_and_write_records(records, stream[0], included=[], valid_relationships=[]) \ No newline at end of file diff --git a/tests/unittests/test_timeout.py b/tests/unittests/test_timeout.py index f71fe69..d77dfa6 100644 --- a/tests/unittests/test_timeout.py +++ b/tests/unittests/test_timeout.py @@ -27,12 +27,12 @@ def test_timeout_value_in_config(self, mocked_parse_args, mocked_request, mocked # get the timeout value for assertion timeout = utils.get_request_timeout() # function call - utils.authed_get("test_source", "", "") + utils.authed_get("test_source", "", "", "") # verify that we got expected timeout value self.assertEquals(100.0, timeout) # verify that the request was called with expected timeout value - mocked_request.assert_called_with(method='get', url='', params='', timeout=100.0) + mocked_request.assert_called_with(method='get', url='', params='', headers='', timeout=100.0) def test_timeout_value_not_in_config(self, mocked_parse_args, mocked_request, mocked_sleep): @@ -43,12 +43,12 @@ def test_timeout_value_not_in_config(self, mocked_parse_args, mocked_request, mo # get the timeout value for assertion timeout = utils.get_request_timeout() # function call - utils.authed_get("test_source", "", "") + utils.authed_get("test_source", "", "","") # verify that we got expected timeout value self.assertEquals(300.0, timeout) # verify that the request was called with expected timeout value - mocked_request.assert_called_with(method='get', url='', params='', timeout=300.0) + mocked_request.assert_called_with(method='get', url='', params='', headers='', timeout=300.0) def test_timeout_string_value_in_config(self, mocked_parse_args, mocked_request, mocked_sleep): @@ -59,12 +59,12 @@ def test_timeout_string_value_in_config(self, mocked_parse_args, mocked_request, # get the timeout value for assertion timeout = utils.get_request_timeout() # function call - utils.authed_get("test_source", "", "") + utils.authed_get("test_source", "", "","") # verify that we got expected timeout value self.assertEquals(100.0, timeout) # verify that the request was called with expected timeout value - mocked_request.assert_called_with(method='get', url='', params='', timeout=100.0) + mocked_request.assert_called_with(method='get', url='', params='', headers='', timeout=100.0) def test_timeout_empty_value_in_config(self, mocked_parse_args, mocked_request, mocked_sleep): @@ -75,12 +75,12 @@ def test_timeout_empty_value_in_config(self, mocked_parse_args, mocked_request, # get the timeout value for assertion timeout = utils.get_request_timeout() # function call - utils.authed_get("test_source", "", "") + utils.authed_get("test_source", "", "","") # verify that we got expected timeout value self.assertEquals(300.0, timeout) # verify that the request was called with expected timeout value - mocked_request.assert_called_with(method='get', url='', params='', timeout=300.0) + mocked_request.assert_called_with(method='get', url='', params='', headers='', timeout=300.0) def test_timeout_0_value_in_config(self, mocked_parse_args, mocked_request, mocked_sleep): @@ -91,12 +91,12 @@ def test_timeout_0_value_in_config(self, mocked_parse_args, mocked_request, mock # get the timeout value for assertion timeout = utils.get_request_timeout() # function call - utils.authed_get("test_source", "", "") + utils.authed_get("test_source", "", "","") # verify that we got expected timeout value self.assertEquals(300.0, timeout) # verify that the request was called with expected timeout value - mocked_request.assert_called_with(method='get', url='', params='', timeout=300.0) + mocked_request.assert_called_with(method='get', url='', params='', headers='', timeout=300.0) def test_timeout_string_0_value_in_config(self, mocked_parse_args, mocked_request, mocked_sleep): @@ -107,12 +107,12 @@ def test_timeout_string_0_value_in_config(self, mocked_parse_args, mocked_reques # get the timeout value for assertion timeout = utils.get_request_timeout() # function call - utils.authed_get("test_source", "", "") + utils.authed_get("test_source", "", "","") # verify that we got expected timeout value self.assertEquals(300.0, timeout) # verify that the request was called with expected timeout value - mocked_request.assert_called_with(method='get', url='', params='', timeout=300.0) + mocked_request.assert_called_with(method='get', url='', params='', headers='', timeout=300.0) @mock.patch("time.sleep") @mock.patch("requests.Session.request") @@ -130,7 +130,7 @@ def test_timeout_backoff(self, mocked_parse_args, mocked_request, mocked_sleep): try: # function call - utils.authed_get("test_source", "", "") + utils.authed_get("test_source", "", "","") except requests.Timeout: pass @@ -148,7 +148,7 @@ def test_connection_error_backoff(self, mocked_parse_args, mocked_request, mocke try: # function call - utils.authed_get("test_source", "", "") + utils.authed_get("test_source", "", "","") except requests.ConnectionError: pass