From 43b85b2765e6fca7a3de561b572399dc80f400ed Mon Sep 17 00:00:00 2001 From: Dylan Sprayberry <28106103+dsprayberry@users.noreply.github.com> Date: Thu, 28 Sep 2023 14:05:07 -0400 Subject: [PATCH] TDL-23255 Upgrade Shopify API to 2023_04 (#178) * upgrade shopify api to 23-04 * try parallel circle tests * Remove fulfillment_service from line_item resource for 2022_10 ----------------------------- Co-authored-by: alu * Updated config.yml * Updated config.yml * Updated config.yml * Updated config.yml * update schema name to avoid test collisions ----------------------------- Co-authored-by: Dylan Sprayberry * decrease parallelism for rate limiting errors ----------------------------- Co-authored-by: Dylan Sprayberry * add fulfillment_service back in since it's nested ----------------------------- Co-authored-by: Dylan Sprayberry * Remove deprecated orders.total_price_usd field for 2022_10 ----------------------------- Co-authored-by: Dylan Sprayberry * Add new orders.merchant_of_record_app_id field ----------------------------- Co-authored-by: Dylan Sprayberry * add new transactions.payment_details fields 2023_01 ----------------------------- Co-authored-by: Dylan Sprayberry * Add new transactions.payment_id field for 2023_01 ----------------------------- Co-authored-by: Dylan Sprayberry * make jsonvalidator happy ----------------------------- Co-authored-by: Dylan Sprayberry * Remove processing_method, payment_details, gateway for 2023_04 ----------------------------- Co-authored-by: Dylan Sprayberry * Add total_unsettled_set object for 2023_04 ----------------------------- Co-authored-by: Dylan Sprayberry * Add current / original fees sets objects for 2023_04 ----------------------------- Co-authored-by: Dylan Sprayberry * Fix json and collapse newly added objects for readability ----------------------------- Co-authored-by: Dylan Sprayberry * version_bump and changelog ----------------------------- Co-authored-by: Dylan Sprayberry * adjust parallelism to 2 so that we don't api calls per second limit ----------------------------- Co-authored-by: Dylan Sprayberry * Fix whitespace --------- Co-authored-by: Leslie VanDeMark Co-authored-by: alu Co-authored-by: Leslie VanDeMark <38043390+leslievandemark@users.noreply.github.com> --- .circleci/config.yml | 84 ++++++++++++++++++- CHANGELOG.md | 5 ++ setup.cfg | 2 - setup.py | 4 +- tap_shopify/__init__.py | 2 +- tap_shopify/schemas/definitions.json | 2 +- tap_shopify/schemas/orders.json | 113 +++++++++++++------------- tap_shopify/schemas/transactions.json | 56 +++++++++++++ tests/test_bookmarks_updated.py | 2 +- 9 files changed, 202 insertions(+), 68 deletions(-) delete mode 100644 setup.cfg diff --git a/.circleci/config.yml b/.circleci/config.yml index 69fbc452..625c2196 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,10 +2,19 @@ version: 2.1 orbs: slack: circleci/slack@3.4.2 -jobs: - build: +executors: + docker-executor: docker: - image: 218546966473.dkr.ecr.us-east-1.amazonaws.com/circle-ci:stitch-tap-tester + +jobs: + build: + executor: docker-executor + steps: + - run: echo "CI Done" + + ensure_env: + executor: docker-executor steps: - checkout - run: @@ -15,16 +24,39 @@ jobs: source /usr/local/share/virtualenvs/tap-shopify/bin/activate pip install -U 'pip<19.2' 'setuptools<51.0.0' pip install .[dev] + - persist_to_workspace: + root: /usr/local/share/virtualenvs + paths: + - tap-shopify + - dev_env.sh + run_pylint: + executor: docker-executor + steps: + - checkout + - attach_workspace: + at: /usr/local/share/virtualenvs - run: name: 'pylint' command: | source /usr/local/share/virtualenvs/tap-shopify/bin/activate pylint tap_shopify -d missing-docstring,too-many-branches + json_validator: + executor: docker-executor + steps: + - checkout + - attach_workspace: + at: /usr/local/share/virtualenvs - run: name: 'JSON Validator' command: | source /usr/local/share/virtualenvs/tap-tester/bin/activate stitch-validate-json tap_shopify/schemas/*.json + run_unit_tests: + executor: docker-executor + steps: + - checkout + - attach_workspace: + at: /usr/local/share/virtualenvs - run: name: 'Unit Tests' command: | @@ -36,6 +68,13 @@ jobs: path: test_output/report.xml - store_artifacts: path: htmlcov + run_integration_tests: + executor: docker-executor + parallelism: 2 + steps: + - checkout + - attach_workspace: + at: /usr/local/share/virtualenvs - run: name: 'Integration Tests' command: | @@ -44,7 +83,14 @@ jobs: mkdir /tmp/${CIRCLE_PROJECT_REPONAME} export STITCH_CONFIG_DIR=/tmp/${CIRCLE_PROJECT_REPONAME} source /usr/local/share/virtualenvs/tap-tester/bin/activate - run-test --tap=tap-shopify tests + circleci tests glob "tests/test_*.py" | circleci tests split > ./tests-to-run + if [ -s ./tests-to-run ]; then + for test_file in $(cat ./tests-to-run) + do + echo $test_file > $STITCH_CONFIG_DIR/tap_test.txt + run-test --tap=${CIRCLE_PROJECT_REPONAME} $test_file + done + fi - slack/notify-on-failure: only_for_branches: master - store_artifacts: @@ -54,10 +100,42 @@ workflows: version: 2 commit: &commit_jobs jobs: + - ensure_env: + context: + - circleci-user + - tier-1-tap-user + - run_pylint: + context: + - circleci-user + - tier-1-tap-user + requires: + - ensure_env + - json_validator: + context: + - circleci-user + - tier-1-tap-user + requires: + - ensure_env + - run_unit_tests: + context: + - circleci-user + - tier-1-tap-user + requires: + - ensure_env + - run_integration_tests: + context: + - circleci-user + - tier-1-tap-user + requires: + - ensure_env - build: context: - circleci-user - tier-1-tap-user + requires: + - run_pylint + - run_unit_tests + - run_integration_tests build_daily: <<: *commit_jobs triggers: diff --git a/CHANGELOG.md b/CHANGELOG.md index e9274846..08abf955 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## 1.8.0 + * Updates the Shopify SDK to 12.3.0 + * Updates API version used to 2023_04 + * Adds and removes fields per Shopify API changelog for versions 2022_10, 2023_01, 2023_04 [#178](https://github.com/singer-io/tap-shopify/pull/178) + ## 1.7.6 * Add backoff for 404 error code [#159](https://github.com/singer-io/tap-shopify/pull/159) diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index b88034e4..00000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -description-file = README.md diff --git a/setup.py b/setup.py index dcfa97d4..b85afa27 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.7.6", + version="1.8.0", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", @@ -11,7 +11,7 @@ python_requires='>=3.5.2', py_modules=["tap_shopify"], install_requires=[ - "ShopifyAPI==12.0.1", + "ShopifyAPI==12.3.0", "singer-python==5.12.1", ], extras_require={ diff --git a/tap_shopify/__init__.py b/tap_shopify/__init__.py index 97faf59f..64803e03 100644 --- a/tap_shopify/__init__.py +++ b/tap_shopify/__init__.py @@ -25,7 +25,7 @@ def initialize_shopify_client(): api_key = Context.config['api_key'] shop = Context.config['shop'] - version = '2022-07' + version = '2023-04' session = shopify.Session(shop, version, api_key) shopify.ShopifyResource.activate_session(session) diff --git a/tap_shopify/schemas/definitions.json b/tap_shopify/schemas/definitions.json index a47a632e..f8e080ff 100644 --- a/tap_shopify/schemas/definitions.json +++ b/tap_shopify/schemas/definitions.json @@ -1481,4 +1481,4 @@ "array" ] } -} \ No newline at end of file +} diff --git a/tap_shopify/schemas/orders.json b/tap_shopify/schemas/orders.json index ed8b533b..26b386ac 100644 --- a/tap_shopify/schemas/orders.json +++ b/tap_shopify/schemas/orders.json @@ -58,12 +58,6 @@ "line_items": { "$ref": "definitions.json#/line_items" }, - "processing_method": { - "type": [ - "null", - "string" - ] - }, "order_number": { "type": [ "null", @@ -414,44 +408,6 @@ ], "format": "singer.decimal" }, - "payment_details": { - "properties": { - "avs_result_code": { - "type": [ - "null", - "string" - ] - }, - "credit_card_company": { - "type": [ - "null", - "string" - ] - }, - "cvv_result_code": { - "type": [ - "null", - "string" - ] - }, - "credit_card_bin": { - "type": [ - "null", - "string" - ] - }, - "credit_card_number": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - }, "number": { "type": [ "null", @@ -574,13 +530,6 @@ "object" ] }, - "total_price_usd": { - "type": [ - "null", - "string" - ], - "format": "singer.decimal" - }, "closed_at": { "type": [ "null", @@ -987,12 +936,6 @@ "integer" ] }, - "gateway": { - "type": [ - "null", - "string" - ] - }, "cart_token": { "type": [ "null", @@ -1278,7 +1221,61 @@ "null", "boolean" ] + }, + "merchant_of_record_app_id": { + "type": ["null", "integer"] + }, + "current_total_additional_fees_set": { + "type": ["null", "object"], + "properties": { + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"], + "format": "singer.decimal"}, + "currency": { + "type": ["null", "string"]} + } + }, + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"], + "format": "singer.decimal"}, + "currency": { + "type": ["null", "string"]} + } + } + } + }, + "original_total_additional_fees_set": { + "type": ["null", "object"], + "properties": { + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"], + "format": "singer.decimal"}, + "currency": { + "type": ["null", "string"] + } + } + }, + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"], + "format": "singer.decimal"}, + "currency": { + "type": ["null", "string"]} + } + } + } } }, "type": "object" -} \ No newline at end of file +} diff --git a/tap_shopify/schemas/transactions.json b/tap_shopify/schemas/transactions.json index 87dcecde..24bfc2fd 100644 --- a/tap_shopify/schemas/transactions.json +++ b/tap_shopify/schemas/transactions.json @@ -111,12 +111,36 @@ "string" ] }, + "credit_card_expiration_month": { + "type": [ + "null", + "integer" + ] + }, + "credit_card_expiration_year": { + "type": [ + "null", + "integer" + ] + }, + "credit_card_name": { + "type": [ + "null", + "string" + ] + }, "credit_card_number": { "type": [ "null", "string" ] }, + "credit_card_wallet": { + "type": [ + "null", + "string" + ] + }, "avs_result_code": { "type": [ "null", @@ -202,6 +226,38 @@ ] } } + }, + "payment_id": { + "type": [ + "null", + "string" + ] + }, + "total_unsettled_set": { + "type": ["null", "object"], + "properties": { + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"], + "format": "singer.decimal"}, + "currency": { + "type": ["null", "string"] + } + } + }, + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"], + "format": "singer.decimal"}, + "currency": { + "type": ["null", "string"]} + } + } + } } }, "type": "object" diff --git a/tests/test_bookmarks_updated.py b/tests/test_bookmarks_updated.py index d590f239..c6d29b3c 100644 --- a/tests/test_bookmarks_updated.py +++ b/tests/test_bookmarks_updated.py @@ -13,7 +13,7 @@ class BookmarkTest(BaseTapTest): """Test tap sets a bookmark and respects it for the next sync of a stream""" @staticmethod def name(): - return "tap_tester_shopify_bookmark_test" + return "tap_tester_shopify_bookmark_update_test" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs)