TDL-23255 Upgrade Shopify API to 2023_04 (#178)

* upgrade shopify api to 23-04 * try parallel circle tests * Remove fulfillment_service from line_item resource for 2022_10 ----------------------------- Co-authored-by: alu <[email protected]> * Updated config.yml * Updated config.yml * Updated config.yml * Updated config.yml * update schema name to avoid test collisions ----------------------------- Co-authored-by: Dylan Sprayberry <[email protected]> * decrease parallelism for rate limiting errors ----------------------------- Co-authored-by: Dylan Sprayberry <[email protected]> * add fulfillment_service back in since it's nested ----------------------------- Co-authored-by: Dylan Sprayberry <[email protected]> * Remove deprecated orders.total_price_usd field for 2022_10 ----------------------------- Co-authored-by: Dylan Sprayberry <[email protected]> * Add new orders.merchant_of_record_app_id field ----------------------------- Co-authored-by: Dylan Sprayberry <[email protected]> * add new transactions.payment_details fields 2023_01 ----------------------------- Co-authored-by: Dylan Sprayberry <[email protected]> * Add new transactions.payment_id field for 2023_01 ----------------------------- Co-authored-by: Dylan Sprayberry <[email protected]> * make jsonvalidator happy ----------------------------- Co-authored-by: Dylan Sprayberry <[email protected]> * Remove processing_method, payment_details, gateway for 2023_04 ----------------------------- Co-authored-by: Dylan Sprayberry <[email protected]> * Add total_unsettled_set object for 2023_04 ----------------------------- Co-authored-by: Dylan Sprayberry <[email protected]> * Add current / original fees sets objects for 2023_04 ----------------------------- Co-authored-by: Dylan Sprayberry <[email protected]> * Fix json and collapse newly added objects for readability ----------------------------- Co-authored-by: Dylan Sprayberry <[email protected]> * version_bump and changelog ----------------------------- Co-authored-by: Dylan Sprayberry <[email protected]> * adjust parallelism to 2 so that we don't api calls per second limit ----------------------------- Co-authored-by: Dylan Sprayberry <[email protected]> * Fix whitespace --------- Co-authored-by: Leslie VanDeMark <[email protected]> Co-authored-by: alu <[email protected]> Co-authored-by: Leslie VanDeMark <[email protected]>
singer-io · Sep 28, 2023 · 43b85b2 · 43b85b2
1 parent ad53f76
commit 43b85b2
Show file tree

Hide file tree

Showing 9 changed files with 202 additions and 68 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -2,10 +2,19 @@ version: 2.1
 orbs:
   slack: circleci/[email protected]
 
-jobs:
-  build:
+executors:
+  docker-executor:
     docker:
       - image: 218546966473.dkr.ecr.us-east-1.amazonaws.com/circle-ci:stitch-tap-tester
+
+jobs:
+  build:
+    executor: docker-executor
+    steps:
+      - run: echo "CI Done"
+
+  ensure_env:
+    executor: docker-executor
     steps:
       - checkout
       - run:
@@ -15,16 +24,39 @@ jobs:
             source /usr/local/share/virtualenvs/tap-shopify/bin/activate
             pip install -U 'pip<19.2' 'setuptools<51.0.0'
             pip install .[dev]
+      - persist_to_workspace:
+          root: /usr/local/share/virtualenvs
+          paths:
+            - tap-shopify
+            - dev_env.sh
+  run_pylint:
+    executor: docker-executor
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /usr/local/share/virtualenvs
       - run:
           name: 'pylint'
           command: |
             source /usr/local/share/virtualenvs/tap-shopify/bin/activate
             pylint tap_shopify -d missing-docstring,too-many-branches
+  json_validator:
+    executor: docker-executor
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /usr/local/share/virtualenvs
       - run:
           name: 'JSON Validator'
           command: |
             source /usr/local/share/virtualenvs/tap-tester/bin/activate
             stitch-validate-json tap_shopify/schemas/*.json
+  run_unit_tests:
+    executor: docker-executor
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /usr/local/share/virtualenvs
       - run:
           name: 'Unit Tests'
           command: |
@@ -36,6 +68,13 @@ jobs:
           path: test_output/report.xml
       - store_artifacts:
           path: htmlcov
+  run_integration_tests:
+    executor: docker-executor
+    parallelism: 2
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /usr/local/share/virtualenvs
       - run:
           name: 'Integration Tests'
           command: |
@@ -44,7 +83,14 @@ jobs:
             mkdir /tmp/${CIRCLE_PROJECT_REPONAME}
             export STITCH_CONFIG_DIR=/tmp/${CIRCLE_PROJECT_REPONAME}
             source /usr/local/share/virtualenvs/tap-tester/bin/activate
-            run-test --tap=tap-shopify tests
+            circleci tests glob "tests/test_*.py" | circleci tests split > ./tests-to-run
+            if [ -s ./tests-to-run ]; then
+              for test_file in $(cat ./tests-to-run)
+              do
+                echo $test_file > $STITCH_CONFIG_DIR/tap_test.txt
+                run-test --tap=${CIRCLE_PROJECT_REPONAME} $test_file
+              done
+            fi
       - slack/notify-on-failure:
           only_for_branches: master
       - store_artifacts:
@@ -54,10 +100,42 @@ workflows:
   version: 2
   commit: &commit_jobs
     jobs:
+      - ensure_env:
+          context:
+            - circleci-user
+            - tier-1-tap-user
+      - run_pylint:
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          requires:
+            - ensure_env
+      - json_validator:
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          requires:
+            - ensure_env
+      - run_unit_tests:
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          requires:
+            - ensure_env
+      - run_integration_tests:
+          context:
+            - circleci-user
+            - tier-1-tap-user
+          requires:
+            - ensure_env
       - build:
           context:
             - circleci-user
             - tier-1-tap-user
+          requires:
+            - run_pylint
+            - run_unit_tests
+            - run_integration_tests
   build_daily:
     <<: *commit_jobs
     triggers:

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,10 @@
 # Changelog
 
+## 1.8.0
+  * Updates the Shopify SDK to 12.3.0
+  * Updates API version used to 2023_04
+  * Adds and removes fields per Shopify API changelog for versions 2022_10, 2023_01, 2023_04 [#178](https://github.com/singer-io/tap-shopify/pull/178)
+
 ## 1.7.6
   * Add backoff for 404 error code [#159](https://github.com/singer-io/tap-shopify/pull/159)
 

diff --git a/setup.cfg b/setup.cfg
diff --git a/setup.py b/setup.py
@@ -3,15 +3,15 @@
 
 setup(
     name="tap-shopify",
-    version="1.7.6",
+    version="1.8.0",
     description="Singer.io tap for extracting Shopify data",
     author="Stitch",
     url="http://github.com/singer-io/tap-shopify",
     classifiers=["Programming Language :: Python :: 3 :: Only"],
     python_requires='>=3.5.2',
     py_modules=["tap_shopify"],
     install_requires=[
-        "ShopifyAPI==12.0.1",
+        "ShopifyAPI==12.3.0",
         "singer-python==5.12.1",
     ],
     extras_require={

diff --git a/tap_shopify/__init__.py b/tap_shopify/__init__.py
@@ -25,7 +25,7 @@
 def initialize_shopify_client():
     api_key = Context.config['api_key']
     shop = Context.config['shop']
-    version = '2022-07'
+    version = '2023-04'
     session = shopify.Session(shop, version, api_key)
     shopify.ShopifyResource.activate_session(session)
 

diff --git a/tap_shopify/schemas/definitions.json b/tap_shopify/schemas/definitions.json
@@ -1481,4 +1481,4 @@
       "array"
     ]
   }
-}
+}
diff --git a/tap_shopify/schemas/orders.json b/tap_shopify/schemas/orders.json
@@ -58,12 +58,6 @@
     "line_items": {
       "$ref": "definitions.json#/line_items"
     },
-    "processing_method": {
-      "type": [
-        "null",
-        "string"
-      ]
-    },
     "order_number": {
       "type": [
         "null",
@@ -414,44 +408,6 @@
       ],
       "format": "singer.decimal"
     },
-    "payment_details": {
-      "properties": {
-        "avs_result_code": {
-          "type": [
-            "null",
-            "string"
-          ]
-        },
-        "credit_card_company": {
-          "type": [
-            "null",
-            "string"
-          ]
-        },
-        "cvv_result_code": {
-          "type": [
-            "null",
-            "string"
-          ]
-        },
-        "credit_card_bin": {
-          "type": [
-            "null",
-            "string"
-          ]
-        },
-        "credit_card_number": {
-          "type": [
-            "null",
-            "string"
-          ]
-        }
-      },
-      "type": [
-        "null",
-        "object"
-      ]
-    },
     "number": {
       "type": [
         "null",
@@ -574,13 +530,6 @@
         "object"
       ]
     },
-    "total_price_usd": {
-      "type": [
-        "null",
-        "string"
-      ],
-      "format": "singer.decimal"
-    },
     "closed_at": {
       "type": [
         "null",
@@ -987,12 +936,6 @@
         "integer"
       ]
     },
-    "gateway": {
-      "type": [
-        "null",
-        "string"
-      ]
-    },
     "cart_token": {
       "type": [
         "null",
@@ -1278,7 +1221,61 @@
         "null",
         "boolean"
       ]
+    },
+    "merchant_of_record_app_id": {
+      "type": ["null", "integer"]
+    },
+    "current_total_additional_fees_set": {
+      "type": ["null", "object"],
+      "properties": {
+        "presentment_money": {
+          "type": ["null", "object"],
+          "properties": {
+            "amount": {
+              "type": ["null", "string"],
+              "format": "singer.decimal"},
+            "currency": {
+              "type": ["null", "string"]}
+          }
+        },
+        "shop_money": {
+          "type": ["null", "object"],
+          "properties": {
+            "amount": {
+              "type": ["null", "string"],
+              "format": "singer.decimal"},
+            "currency": {
+              "type": ["null", "string"]}
+          }
+        }
+      }
+    },
+    "original_total_additional_fees_set": {
+      "type": ["null", "object"],
+      "properties": {
+        "presentment_money": {
+          "type": ["null", "object"],
+          "properties": {
+            "amount": {
+              "type": ["null", "string"],
+              "format": "singer.decimal"},
+            "currency": {
+              "type": ["null", "string"]
+            }
+          }
+        },
+        "shop_money": {
+          "type": ["null", "object"],
+          "properties": {
+            "amount": {
+              "type": ["null", "string"],
+              "format": "singer.decimal"},
+            "currency": {
+              "type": ["null", "string"]}
+          }
+        }
+      }
     }
   },
   "type": "object"
-}
+}
-Original file line number
+Diff line change
@@ Expand Up / @@ -1481,4 +1481,4 @@ @@
           "array"
         ]
       }
-    }
+    }