Skip to content

Commit

Permalink
feat: configurable exponential backoff for webhooks retry
Browse files Browse the repository at this point in the history
  • Loading branch information
@ptitFicus
ptitFicus committed Jun 15, 2024
1 parent 8b0c151 commit 6a2ce14
Show file tree
Hide file tree
Showing 11 changed files with 441 additions and 276 deletions.
60 changes: 36 additions & 24 deletions app/fr/maif/izanami/jobs/WebhookListener.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,28 @@ import scala.concurrent.duration._
import scala.concurrent.{ExecutionContext, Future}

class WebhookListener(env: Env, eventService: EventService) {
private val handlebars = new Handlebars()
private val mapper = new ObjectMapper()
private val logger = env.logger
private implicit val ec: ExecutionContext = env.executionContext
private implicit val actorSystem: ActorSystem = env.actorSystem
private val handlebars = new Handlebars()
private val mapper = new ObjectMapper()
private val logger = env.logger
private implicit val ec: ExecutionContext = env.executionContext
private implicit val actorSystem: ActorSystem = env.actorSystem
private val cancelSwitch: Map[String, Cancellable] = Map()
private val retryCount: Long = env.configuration.getOptional[Long]("app.webhooks.retry.count").getOrElse {
logger.warn("Failed to parse app.webhooks.retry.count as long, will use 5 as default value")
5L
}
private val retryInitialDelay = env.configuration.getOptional[Long]("app.webhooks.retry.intial-delay").getOrElse {
logger.warn("Failed to parse app.webhooks.retry.intial-delay as long, will use 5 as default value")
5L
}
private val retryMaxDelay = env.configuration.getOptional[Long]("app.webhooks.retry.max-delay").getOrElse {
logger.warn("Failed to parse app.webhooks.retry.max-delay as long, will use 600 as default value")
600L
}
private val retryMultiplier = env.configuration.getOptional[Long]("app.webhooks.retry.multiplier").getOrElse {
logger.warn("Failed to parse app.webhooks.retry.multiplier as long, will use 2 as default value")
2L
}

def onStart(): Future[Unit] = {
env.datastores.tenants
Expand All @@ -43,16 +59,14 @@ class WebhookListener(env: Env, eventService: EventService) {
def handleGlobalEvent(event: IzanamiEvent): Unit = {
event match {
case TenantCreated(eventId, tenant, _) => startListening(tenant)
case TenantDeleted(_, tenant, _) => cancelSwitch.get(tenant).map(c => c.cancel())
case _ => ()
case TenantDeleted(_, tenant, _) => cancelSwitch.get(tenant).map(c => c.cancel())
case _ => ()
}
}



def startListening(tenant: String): Unit = {
logger.info(s"Initializing webhook event listener for tenant $tenant")
val cancelRef = new AtomicReference[Cancellable]()
val cancelRef = new AtomicReference[Cancellable]()
val cancellable = env.actorSystem.scheduler.scheduleAtFixedRate(0.minutes, 5.minutes)(() => {
env.datastores.webhook
.findAbandoneddWebhooks(tenant)
Expand All @@ -66,7 +80,7 @@ class WebhookListener(env: Env, eventService: EventService) {
handleEventForHook(tenant, event.asInstanceOf[FeatureEvent], webhook)
}
}
case None => cancelRef.get().cancel()
case None => cancelRef.get().cancel()
}
})

Expand Down Expand Up @@ -138,25 +152,23 @@ class WebhookListener(env: Env, eventService: EventService) {
private def futureWithRetry[T](
expression: () => Future[T],
onFailure: () => Future[Any],
factor: Float = 1.5f,
previousWait: FiniteDuration = 5.seconds,
max: Int = 5,
cur: Int = 0
)(implicit as: ActorSystem): Future[T] = {
expression().recoverWith { case e =>
onFailure().flatMap(_ => {
logger.error(s"Call failed", e)
val next: FiniteDuration =
if (cur == 0) previousWait
else (Math.round(previousWait.toMillis * factor)).milliseconds
var next: FiniteDuration = Math.round(retryInitialDelay * 1000 * (Math.pow(retryMultiplier, cur))).milliseconds
if(next > retryMaxDelay.seconds) {
next = retryMaxDelay.seconds
}

if (cur >= max) {
logger.error(s"Exceeded max retry ($max), stopping...")
if (cur >= retryCount) {
logger.error(s"Exceeded max retry ($retryCount), stopping...")
Future.failed(WebhookRetryCountExceeded())
} else {
logger.error(s"Will retry after ${next.toSeconds} seconds (${cur + 1} / $max)")
logger.error(s"Will retry after ${next.toSeconds} seconds (${cur + 1} / $retryCount)")
after(next, as.scheduler, global, () => Future.successful(1)).flatMap { _ =>
futureWithRetry(expression, onFailure, factor, next, max, cur + 1)(actorSystem)
futureWithRetry(expression, onFailure, cur + 1)(actorSystem)
}
}
})
Expand All @@ -182,10 +194,10 @@ class WebhookListener(env: Env, eventService: EventService) {

private def callWebhook(webhook: LightWebhook, body: String): Future[Unit] = {
logger.info(s"Calling ${webhook.url.toString}")
val hasContentType = webhook.headers.exists {
case (name, _) => name.equalsIgnoreCase("Content-Type")
val hasContentType = webhook.headers.exists { case (name, _) =>
name.equalsIgnoreCase("Content-Type")
}
val headers = if(hasContentType) {
val headers = if (hasContentType) {
webhook.headers
} else {
webhook.headers + ("Content-Type" -> "application/json")
Expand Down
12 changes: 12 additions & 0 deletions conf/application.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,18 @@ app {
url = "https://reporting.otoroshi.io/izanami/ingest"
url = ${?IZANAMI_REPORTING_URL}
}
webhooks {
retry {
count = 5
count = ${?IZANAMI_WEBHOOK_RETRY_COUNT}
intial-delay = 5
intial-delay = ${?IZANAMI_WEBHOOK_RETRY_INITIAL_DELAY}
max-delay = 600
max-delay = ${?IZANAMI_WEBHOOK_RETRY_MAX_DELAY}
multiplier = 2
multiplier = ${?IZANAMI_WEBHOOK_RETRY_MULTIPLIER}
}
}
wasm {
cache {
ttl = 60000
Expand Down
10 changes: 5 additions & 5 deletions manual/.docusaurus/@easyops-cn/docusaurus-search-local/default/generated.js
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
import lunr from "/home/runner/work/izanami/izanami/manual/node_modules/lunr/lunr.js";
require("/home/runner/work/izanami/izanami/manual/node_modules/lunr-languages/lunr.stemmer.support.js")(lunr);
import lunr from "/Users/77199M/workspace/oss/izanami-v2/manual/node_modules/lunr/lunr.js";
require("/Users/77199M/workspace/oss/izanami-v2/manual/node_modules/lunr-languages/lunr.stemmer.support.js")(lunr);
require("@easyops-cn/docusaurus-search-local/dist/client/shared/lunrLanguageZh").lunrLanguageZh(lunr);
require("/home/runner/work/izanami/izanami/manual/node_modules/lunr-languages/lunr.multi.js")(lunr);
require("/Users/77199M/workspace/oss/izanami-v2/manual/node_modules/lunr-languages/lunr.multi.js")(lunr);
export const language = ["en","zh"];
export const removeDefaultStopWordFilter = false;
export const removeDefaultStemmer = false;
export { default as Mark } from "/home/runner/work/izanami/izanami/manual/node_modules/mark.js/dist/mark.js"
export const searchIndexUrl = "search-index{dir}.json?_=2de5ae58";
export { default as Mark } from "/Users/77199M/workspace/oss/izanami-v2/manual/node_modules/mark.js/dist/mark.js"
export const searchIndexUrl = "search-index{dir}.json?_=11a1c7b8";
export const searchResultLimits = 8;
export const searchResultContextMaxLength = 50;
export const explicitSearchResultPath = true;
Expand Down
8 changes: 4 additions & 4 deletions manual/.docusaurus/client-modules.js
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
export default [
require("/home/runner/work/izanami/izanami/manual/node_modules/infima/dist/css/default/default.css"),
require("/home/runner/work/izanami/izanami/manual/node_modules/@docusaurus/theme-classic/lib/prism-include-languages"),
require("/home/runner/work/izanami/izanami/manual/node_modules/@docusaurus/theme-classic/lib/nprogress"),
require("/home/runner/work/izanami/izanami/manual/src/css/custom.css"),
require("/Users/77199M/workspace/oss/izanami-v2/manual/node_modules/infima/dist/css/default/default.css"),
require("/Users/77199M/workspace/oss/izanami-v2/manual/node_modules/@docusaurus/theme-classic/lib/prism-include-languages"),
require("/Users/77199M/workspace/oss/izanami-v2/manual/node_modules/@docusaurus/theme-classic/lib/nprogress"),
require("/Users/77199M/workspace/oss/izanami-v2/manual/src/css/custom.css"),
];
38 changes: 34 additions & 4 deletions manual/.docusaurus/docusaurus-plugin-debug/default/docusaurus-debug-all-content-673.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@
},
"sidebar": "tutorialSidebar",
"previous": {
"title": "Configuring Izanami",
"permalink": "/izanami/docs/guides/configuration"
"title": "Webhooks",
"permalink": "/izanami/docs/guides/webhooks"
},
"next": {
"title": "Java client",
Expand Down Expand Up @@ -235,8 +235,8 @@
"permalink": "/izanami/docs/guides/mailer-configuration"
},
"next": {
"title": "Clients",
"permalink": "/izanami/docs/clients/"
"title": "Webhooks",
"permalink": "/izanami/docs/guides/webhooks"
}
},
{
Expand Down Expand Up @@ -498,6 +498,32 @@
"permalink": "/izanami/docs/guides/key-configuration"
}
},
{
"id": "guides/webhooks",
"title": "Webhooks",
"description": "Webhooks are a way to make Izanami call a provided URL when a feature is modified.",
"source": "@site/docs/04-guides/12-webhooks.mdx",
"sourceDirName": "04-guides",
"slug": "/guides/webhooks",
"permalink": "/izanami/docs/guides/webhooks",
"draft": false,
"unlisted": false,
"tags": [],
"version": "current",
"sidebarPosition": 12,
"frontMatter": {
"title": "Webhooks"
},
"sidebar": "tutorialSidebar",
"previous": {
"title": "Configuring Izanami",
"permalink": "/izanami/docs/guides/configuration"
},
"next": {
"title": "Clients",
"permalink": "/izanami/docs/clients/"
}
},
{
"id": "usages/contexts",
"title": "Contexts",
Expand Down Expand Up @@ -718,6 +744,10 @@
{
"type": "doc",
"id": "guides/configuration"
},
{
"type": "doc",
"id": "guides/webhooks"
}
],
"link": {
Expand Down
Loading

0 comments on commit 6a2ce14

Please sign in to comment.