Crest Work

.circleci/config.yml

@@ -17,17 +17,17 @@ jobs:
           command: |
             source /usr/local/share/virtualenvs/tap-tester/bin/activate
             stitch-validate-json tap_quickbooks/schemas/*.json
-      #- run:
-      #    name: 'pylint'
-      #    command: |
-      #      source /usr/local/share/virtualenvs/tap-quickbooks/bin/activate
-      #      # TODO: Adjust the pylint disables
-      #      #pylint tap_quickbooks --disable 'broad-except,chained-comparison,empty-docstring,fixme,invalid-name,line-too-long,missing-class-docstring,missing-function-docstring,missing-module-docstring,no-else-raise,no-else-return,too-few-public-methods,too-many-arguments,too-many-branches,too-many-lines,too-many-locals,ungrouped-imports,wrong-spelling-in-comment,wrong-spelling-in-docstring,bad-whitespace'
+      - run:
+         name: 'pylint'
+         command: |
+            source /usr/local/share/virtualenvs/tap-quickbooks/bin/activate
+            # TODO: Adjust the pylint disables
+            pylint tap_quickbooks --disable 'broad-except,chained-comparison,empty-docstring,fixme,invalid-name,line-too-long,missing-class-docstring,missing-function-docstring,missing-module-docstring,no-else-raise,no-else-return,too-few-public-methods,too-many-arguments,too-many-branches,too-many-lines,too-many-locals,ungrouped-imports,wrong-spelling-in-comment,wrong-spelling-in-docstring,bad-whitespace,undefined-loop-variable'
       - run:
           name: 'Unit Tests'
           command: |
             source /usr/local/share/virtualenvs/tap-quickbooks/bin/activate
-            pip install nose coverage
+            pip install nose coverage parameterized
             nosetests --with-coverage --cover-erase --cover-package=tap_quickbooks --cover-html-dir=htmlcov tests/unittests
             coverage html
       - store_test_results:

README.md

@@ -65,9 +65,13 @@ These values are all obtained from the oauth steps documented on [quickbook's do
     $ pip install -e .
     ```
 1. Create your tap's `config.json` file.  The tap config file for this tap should include these entries:
-   - `start_date` - the default value to use if no bookmark exists for an endpoint (rfc3339 date string)
-   - `user_agent` (string, optional): Process and email for API logging purposes. Example: `tap-quickbooks <api_user_email@your_company.com>`
-   - `sandbox` (string, optional): Whether to communication with quickbooks's sandbox or prod account for this application. If you're not sure leave out. Defaults to false.
+   - `start_date` - The default value to use if no bookmark exists for an endpoint (rfc3339 date string)
+   - `user_agent` (string): Process and email for API logging purposes. Example: `tap-quickbooks <api_user_email@your_company.com>`
+   - `realm_id` (string): The realm id of the company to fetch the data from.
+   - `client_secret` (string): Credentials of the client app.
+   - `client_id` (string): Id of the client app.
+   - `refresh_token` (string): Token to get a new Access token if it expires.
+   - `sandbox` (string, optional): Whether to communicate with quickbooks's sandbox or prod account for this application. If you're not sure leave out. Defaults to false.
    - The `request_timeout` is an optional paramater to set timeout for requests. Default: 300 seconds
 
    And the other values mentioned in [the authentication section above](#authentication).
@@ -78,9 +82,10 @@ These values are all obtained from the oauth steps documented on [quickbook's do
 		"start_date": "2020-08-21T00:00:00Z",
 		"refresh_token": "<refresh_token>",
 		"client_secret": "<app_secret>",
+		"realm_id": "0123456789",
 		"sandbox": "<true|false>",
 		"user_agent": "Stitch Tap ([email protected])",
-        	"request_timeout": 300
+		"request_timeout": 300
 	}
 	```
 

tap_quickbooks/__init__.py

@@ -11,12 +11,11 @@
 
 @utils.handle_top_exception(LOGGER)
 def main():
-    required_config_keys = ['start_date']
+    required_config_keys = ['start_date', 'user_agent', 'realm_id', 'client_id', 'client_secret', 'refresh_token']
     args = singer.parse_args(required_config_keys)
 
     config = args.config
     client = QuickbooksClient(args.config_path, config)
-    catalog = args.catalog or Catalog([])
     state = args.state
 
     if args.properties and not args.catalog:
@@ -26,9 +25,16 @@ def main():
         LOGGER.info("Starting discovery mode")
         catalog = do_discover()
         write_catalog(catalog)
+        LOGGER.info("Finished discovery mode")
     else:
+        if args.catalog:
+            catalog = args.catalog
+        else:
+            catalog = do_discover()
+
         LOGGER.info("Starting sync mode")
         do_sync(client, config, state, catalog)
+        LOGGER.info("Finished sync mode")
 
 if __name__ == "__main__":
     main()

tap_quickbooks/client.py

@@ -14,16 +14,103 @@
 SANDBOX_ENDPOINT_BASE = "https://sandbox-quickbooks.api.intuit.com"
 TOKEN_REFRESH_URL = 'https://oauth.platform.intuit.com/oauth2/v1/tokens/bearer'
 
-
-class QuickbooksAuthenticationError(Exception):
-    pass
+class QuickbooksError(Exception):
+    """Base class for Quickbooks exceptions"""
 
 class Quickbooks5XXException(Exception):
-    pass
+    """Base class for 5XX errors"""
 
 class Quickbooks4XXException(Exception):
-    pass
+    """Base class for 4XX errors"""
+
+class QuickbooksAuthenticationError(Quickbooks4XXException):
+    """Exception class for 401 error"""
+
+class QuickbooksBadRequestError(Quickbooks4XXException):
+    """Exception class for 400 error"""
+
+class QuickbooksForbiddenError(Quickbooks4XXException):
+    """Exception class for 403 error"""
+
+class QuickbooksNotFoundError(Quickbooks4XXException):
+    """Exception class for 404 error"""
+
+class QuickbooksInternalServerError(Quickbooks5XXException):
+    """Exception class for 500 error"""
+
+class QuickbooksServiceUnavailableError(Quickbooks5XXException):
+    """Exception class for 503 error"""
+
+# Documentation: https://developer.intuit.com/app/developer/qbo/docs/develop/troubleshooting/error-codes
+ERROR_CODE_EXCEPTION_MAPPING = {
+    400: {
+        "exception": QuickbooksBadRequestError,
+        "message": "The request can't be fulfilled due to bad syntax."
+    },
+    401: {
+        "exception": QuickbooksAuthenticationError,
+        "message": "Authentication or authorization failed. Usually, this means the token in use won't work for API calls since it's either expired or revoked."
+    },
+    403: {
+        "exception": QuickbooksForbiddenError,
+        "message": "The URL exists, but it's restricted. External developers can't use or consume resources from this URL."
+    },
+    404: {
+        "exception": QuickbooksNotFoundError,
+        "message": "Couldn't find the requested resource or URL, or it doesn't exist."
+    },
+    500: {
+        "exception": QuickbooksInternalServerError,
+        "message": "A server error occurred while processing the request."
+    },
+    503: {
+        "exception": QuickbooksServiceUnavailableError,
+        "message": "The service is temporarily unavailable at the Server side.",
+    }
+}
+
+def get_exception_for_error_code(error_code):
+    """Function to retrieve exceptions based on error_code"""
+
+    exception = ERROR_CODE_EXCEPTION_MAPPING.get(error_code, {}).get('exception')
+    # If the error code is not from the listed error codes then return Quickbooks5XXException, Quickbooks4XXException or QuickbooksError respectively
+    if not exception:
+        if error_code >= 500:
+            return Quickbooks5XXException
+        if error_code >= 400:
+            return Quickbooks4XXException
+        return QuickbooksError
+    return exception
+
+def raise_for_error(response):
+    """Function to raise an error by extracting the message from the error response"""
+
+    error_code = response.status_code
+
+    try:
+        response_json = response.json()
+    except Exception:
+        response_json = {}
+
+    if response_json.get('Fault', response_json.get('fault')):
+
+        errors = response_json.get('Fault', {}).get('Error', response_json.get('fault', {}).get('error'))
+        # Prepare the message with detail if there is single error
+        if len(errors) == 1:
+            errors = errors[0]
+            msg = errors.get('Message', errors.get('message'))
+            detail = errors.get('Detail', errors.get('detail'))
+            internal_error_code = errors.get('code')
+            error_message  = 'Message: {}, Quickbooks Error code: {}, Detail: {}'.format(msg, internal_error_code, detail)
+        else:
+            error_message = errors
+    else:
+        error_message = ERROR_CODE_EXCEPTION_MAPPING.get(error_code, {}).get('message', 'Unknown Error')
+
+    message = 'HTTP-error-code: {}, Error: {}'.format(error_code, error_message)
+    ex = get_exception_for_error_code(error_code)
 
+    raise ex(message) from None
 
 class QuickbooksClient():
     def __init__(self, config_path, config):
@@ -52,12 +139,14 @@ def __init__(self, config_path, config):
                                      auto_refresh_url=TOKEN_REFRESH_URL,
                                      auto_refresh_kwargs=extra,
                                      token_updater=self._write_config)
+        # Latest minorversion is '65' according to doc, https://developer.intuit.com/app/developer/qbo/docs/learn/explore-the-quickbooks-online-api/minor-versions
+        self.minor_version = 65
         try:
-            # Make an authenticated request after creating the object to any endpoint
-            self.get('/v3/company/{}/query'.format(self.realm_id), params={"query": "SELECT * FROM CompanyInfo"})
+            # Make an authenticated request to any endpoint with minorversion=65 to validate the client object
+            self.get('/v3/company/{}/query'.format(self.realm_id), params={"query": "SELECT * FROM CompanyInfo", "minorversion": self.minor_version})
         except Exception as e:
             LOGGER.info("Error initializing QuickbooksClient during token refresh, please reauthenticate.")
-            raise QuickbooksAuthenticationError(e)
+            raise e
 
     def _write_config(self, token):
         LOGGER.info("Credentials Refreshed")
@@ -71,12 +160,10 @@ def _write_config(self, token):
         with open(self.config_path, 'w') as file:
             json.dump(config, file, indent=2)
 
-    # Backoff th request for 5 times when Timeout error occurs
     @backoff.on_exception(backoff.expo, Timeout, max_tries=5, factor=2)
     @backoff.on_exception(backoff.constant,
                           (Quickbooks5XXException,
                            Quickbooks4XXException,
-                           QuickbooksAuthenticationError,
                            requests.ConnectionError),
                           max_tries=3,
                           interval=10)
@@ -116,12 +203,8 @@ def _make_request(self, method, endpoint, headers=None, params=None, data=None):
         response = self.session.request(method, full_url, headers=headers, params=params, data=data, timeout = request_timeout)
 
         # TODO: Check error status, rate limit, etc.
-        if response.status_code >= 500:
-            raise Quickbooks5XXException(response.text)
-        elif response.status_code in (401, 403):
-            raise QuickbooksAuthenticationError(response.text)
-        elif response.status_code >= 400:
-            raise Quickbooks4XXException(response.text)
+        if response.status_code != 200:
+            raise_for_error(response)
 
         return response.json()
 

tap_quickbooks/discover.py

@@ -3,8 +3,8 @@
 
 from singer import metadata
 from singer.catalog import Catalog
-from .streams import STREAM_OBJECTS
 import singer
+from .streams import STREAM_OBJECTS
 
 LOGGER = singer.get_logger()
 
@@ -26,7 +26,7 @@ def _load_schemas():
             try:
                 schemas[file_raw] = json.load(file)
             except:
-                LOGGER.info('Failed to load file {}'.format(file_raw))
+                LOGGER.info('Failed to load file %s', file_raw)
                 raise
 
     return schemas
@@ -61,7 +61,8 @@ def do_discover():
                 key_properties=stream.key_properties,
                 valid_replication_keys=stream.replication_keys,
                 replication_method=stream.replication_method,
-        ))
+            )
+        )
         # Set the replication_key MetaData to automatic as well
         mdata = metadata.write(mdata, ('properties', stream.replication_keys[0]), 'inclusion', 'automatic')
         # load all refs

tap_quickbooks/schemas/credit_memos.json

@@ -30,9 +30,10 @@
       ]
     },
     "Balance": {
+      "format": "singer.decimal",
       "type": [
         "null",
-        "integer"
+        "string"
       ]
     },
     "CustomerMemo": {
@@ -153,57 +154,7 @@
       "$ref": "shared/ref_schema.json"
     },
     "TxnTaxDetail": {
-      "properties": {
-        "TotalTax": {
-          "type": ["null", "integer"]
-        },
-        "TxnTaxCodeRef": {
-          "$ref": "shared/ref_schema.json"
-        },
-        "TaxLine": {
-          "items": {
-            "properties": {
-              "DetailType": {
-                "type": ["null", "string"]
-              },
-              "Amount": {
-                "format": "singer.decimal",
-                "type": ["null", "string"]
-              },
-              "TaxLineDetail": {
-                "properties": {
-                  "TaxRateRef": {
-                    "$ref": "shared/ref_schema.json"
-                  },
-                  "NetAmountTaxable": {
-                    "format": "singer.decimal",
-                    "type": ["null", "string"]
-                  },
-                  "PercentBased": {
-                    "type": ["null", "boolean"]
-                  },
-                  "TaxInclusiveAmount": {
-                    "format": "singer.decimal",
-                    "type": ["null", "string"]
-                  },
-                  "OverrideDeltaAmount": {
-                    "format": "singer.decimal",
-                    "type": ["null", "string"]
-                  },
-                  "TaxPercent": {
-                    "format": "singer.decimal",
-                    "type": ["null", "string"]
-                  }
-                },
-                "type": ["null", "object"]
-              }
-            },
-            "type": ["null", "object"]
-          },
-          "type": ["null", "array"]
-        }
-      },
-      "type": ["null", "object"]
+      "$ref": "shared/txn_tax_detail.json"
     },
     "TxnDate": {
       "format": "date-time",