Merge pull request #9 from rwxd/init

feat(limits): added ratelimits and output improvements

README.md

@@ -21,7 +21,7 @@ pipx install wallabag2readwise
 ### Commands
 
 ```bash
-wallabag2readwise push
+wallabag2readwise
 ```
 
 ### Configuration

requirements.txt

@@ -1,3 +1,5 @@
 typer==0.7.0
 rich==12.6.0
 requests==2.28.1
+ratelimit==2.2.1
+backoff==2.2.1

wallabag2readwise/logging.py

@@ -1,7 +1,7 @@
 import logging
 from os import getenv
 
-log_level = getenv('LOG_LEVEL', 'DEBUG')
+log_level = getenv('LOG_LEVEL', 'WARNING')
 logger = logging.getLogger('wallabag2readwise')
 
 format = '%(levelname)s - %(asctime)s - %(name)s - %(message)s'

wallabag2readwise/misc.py

@@ -3,6 +3,7 @@
 from wallabag2readwise.wallabag import WallabagConnector
 from datetime import datetime
 from wallabag2readwise.logging import logger
+from wallabag2readwise.output import console
 
 
 def push_annotations(wallabag: WallabagConnector, readwise: ReadwiseConnector):
@@ -28,15 +29,17 @@ def push_annotations(wallabag: WallabagConnector, readwise: ReadwiseConnector):
             for article in readwise_articles:
                 if article.title == entry.title:
                     highlights = list(readwise.get_book_highlights(article.id))
-                    print(f'Found {len(highlights)} for {entry.title} in Readwise')
+                    console.print(
+                        f'=> Found {len(highlights)} Readwise highlights for "{entry.title}"'
+                    )
                     for annotation in annotations:
                         if annotation.quote not in [i.text for i in highlights]:
-                            logger.debug('Adding annotation')
                             new_highlights(readwise, entry, [annotation])
                         else:
                             logger.debug('Annotation already present')
 
                     break
             else:
                 logger.info(f'Entry "{entry.title}" not present in Readwise')
+                console.print(f'==> Adding article "{entry.title}"')
                 new_highlights(readwise, entry, annotations)

wallabag2readwise/output.py

@@ -0,0 +1,3 @@
+from rich.console import Console
+
+console = Console()

wallabag2readwise/readwise.py

@@ -3,8 +3,15 @@
 from typing import Generator
 from datetime import datetime
 from typing import Optional
+from ratelimit import limits, RateLimitException
+from backoff import on_exception, expo
 
 from wallabag2readwise.models import Annotation, Entry, ReadwiseBook, ReadwiseHighlight
+from wallabag2readwise.output import console
+
+
+class ReadwiseRateLimitException(Exception):
+    pass
 
 
 class ReadwiseConnector:
@@ -26,17 +33,35 @@ def _session(self) -> requests.Session:
         )
         return session
 
-    def get(self, endpoint: str, params: dict = {}) -> requests.Response:
+    @on_exception(expo, ReadwiseRateLimitException, max_tries=8)
+    @on_exception(expo, RateLimitException, max_tries=8)
+    @limits(calls=230, period=60)
+    def _request(
+        self, method: str, endpoint: str, params: dict = {}, data: dict = {}
+    ) -> requests.Response:
         url = self.url + endpoint
-        logger.debug(f'Getting "{url}" with params: {params}')
-        response = self._session.get(url, params=params)
+        logger.debug(f'Calling "{method}" on "{url}" with params: {params}')
+        response = self._session.request(method, url, params=params, json=data)
+        if response.status_code == 429:
+            raise ReadwiseRateLimitException()
         response.raise_for_status()
         return response
 
+    def get(self, endpoint: str, params: dict = {}) -> requests.Response:
+        logger.debug(f'Getting "{endpoint}" with params: {params}')
+        response = self._request('GET', endpoint, params=params)
+        return response
+
+    @on_exception(expo, RateLimitException, max_tries=8)
+    @limits(calls=19, period=60)
+    def get_with_limit_19(self, endpoint: str, params: dict = {}) -> requests.Response:
+        response = self.get(endpoint, params)
+        return response
+
     def post(self, endpoint: str, data: dict = {}) -> requests.Response:
         url = self.url + endpoint
         logger.debug(f'Posting "{url}" with data: {data}')
-        response = self._session.post(url, json=data)
+        response = self._request('POST', endpoint, data=data)
         response.raise_for_status()
         return response
 
@@ -103,6 +128,7 @@ def new_highlights(
     readwise: ReadwiseConnector, entry: Entry, annotations: list[Annotation]
 ):
     for item in annotations:
+        console.print(f'==> Adding highlight')
         readwise.create_highlight(
             item.quote,
             entry.title,