Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Azure Synapse Workspace and Pools example #501

Merged
merged 2 commits into from
Aug 22, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions examples/azure-synapse/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
### Scala an JVM
*.class
*.log
.bsp
.scala-build

# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
hs_err_pid*

kubeconfig.json
125 changes: 125 additions & 0 deletions examples/azure-synapse/Main.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
import besom.*
import besom.api.azurenative
import besom.api.random

@main def main = Pulumi.run {

val resourceGroup = azurenative.resources.ResourceGroup("synapse-rg")

val storageAccount = azurenative.storage.StorageAccount(
name = "synapsesa",
azurenative.storage.StorageAccountArgs(
resourceGroupName = resourceGroup.name,
accessTier = azurenative.storage.enums.AccessTier.Hot,
enableHttpsTrafficOnly = true,
isHnsEnabled = true,
kind = azurenative.storage.enums.Kind.StorageV2,
sku = azurenative.storage.inputs.SkuArgs(
name = azurenative.storage.enums.SkuName.Standard_RAGRS
)
)
)

val users = azurenative.storage.BlobContainer(
name = "users",
azurenative.storage.BlobContainerArgs(
resourceGroupName = resourceGroup.name,
accountName = storageAccount.name,
publicAccess = None
)
)

val dataLakeStorageAccountUrl = p"https://${storageAccount.name}.dfs.core.windows.net"

val workspace = azurenative.synapse.Workspace(
name = "my-workspace",
azurenative.synapse.WorkspaceArgs(
resourceGroupName = resourceGroup.name,
defaultDataLakeStorage = azurenative.synapse.inputs.DataLakeStorageAccountDetailsArgs(
accountUrl = dataLakeStorageAccountUrl,
filesystem = "users"
),
identity = azurenative.synapse.inputs.ManagedIdentityArgs(
`type` = azurenative.synapse.enums.ResourceIdentityType.SystemAssigned
),
sqlAdministratorLogin = "sqladminuser",
sqlAdministratorLoginPassword = random.RandomPassword("workspacePwd", random.RandomPasswordArgs(length = 12)).result
)
)

val firewallRule = azurenative.synapse.IpFirewallRule(
name = "allowAll",
azurenative.synapse.IpFirewallRuleArgs(
resourceGroupName = resourceGroup.name,
workspaceName = workspace.name,
endIpAddress = "255.255.255.255",
startIpAddress = "0.0.0.0"
)
)

val subscriptionId = resourceGroup.id.map(_.split("/")(2))
val roleDefinitionId =
p"/subscriptions/$subscriptionId/providers/Microsoft.Authorization/roleDefinitions/ba92f5b4-2d11-453d-a403-e96b0029c9fe"

val storageAccess = azurenative.authorization.RoleAssignment(
name = "storageAccess",
azurenative.authorization.RoleAssignmentArgs(
roleAssignmentName = random.RandomUuid("roleName").result,
scope = storageAccount.id,
principalId = workspace.identity.principalId.getOrElse("<preview>"),
principalType = azurenative.authorization.enums.PrincipalType.ServicePrincipal,
roleDefinitionId = roleDefinitionId
)
)

val clientConfig = azurenative.authorization.getClientConfig()

val userAccess = azurenative.authorization.RoleAssignment(
name = "userAccess",
azurenative.authorization.RoleAssignmentArgs(
roleAssignmentName = random.RandomUuid("userRoleName").result,
scope = storageAccount.id,
principalId = clientConfig.objectId,
principalType = azurenative.authorization.enums.PrincipalType.User,
roleDefinitionId = roleDefinitionId
)
)

val sqlPool = azurenative.synapse.SqlPool(
name = "SQLPOOL1",
azurenative.synapse.SqlPoolArgs(
resourceGroupName = resourceGroup.name,
workspaceName = workspace.name,
collation = "SQL_Latin1_General_CP1_CI_AS",
createMode = azurenative.synapse.enums.CreateMode.Default,
sku = azurenative.synapse.inputs.SkuArgs(
name = "DW100c"
)
)
)

val sparkPool = azurenative.synapse.BigDataPool(
name = "Spark1",
azurenative.synapse.BigDataPoolArgs(
resourceGroupName = resourceGroup.name,
workspaceName = workspace.name,
autoPause = azurenative.synapse.inputs.AutoPausePropertiesArgs(
delayInMinutes = 15,
enabled = true
),
autoScale = azurenative.synapse.inputs.AutoScalePropertiesArgs(
enabled = true,
maxNodeCount = 3,
minNodeCount = 3
),
nodeCount = 3,
nodeSize = azurenative.synapse.enums.NodeSize.Small,
nodeSizeFamily = azurenative.synapse.enums.NodeSizeFamily.MemoryOptimized,
sparkVersion = "3.3"
)
)

Stack(users, firewallRule, storageAccess, userAccess, sqlPool, sparkPool).exports(
sparkPoolName = sparkPool.name
)
}
3 changes: 3 additions & 0 deletions examples/azure-synapse/Pulumi.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
name: azure-synapse
description: Azure Synapse example
runtime: scala
43 changes: 43 additions & 0 deletions examples/azure-synapse/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# Azure Synapse Workspace and Pools

Starting point for enterprise analytics solutions based on Azure Synapse.

## Deploying the App

To deploy your infrastructure, follow the below steps.

### Prerequisites

1. [Install Pulumi](https://www.pulumi.com/docs/get-started/install/)
2. [Configure Azure Credentials](https://www.pulumi.com/docs/intro/cloud-providers/azure/setup/)

## Running the App

1. Create a new stack:

```bash
$ pulumi stack init dev
```

2. Set the Azure region location to use:

```bash
$ pulumi config set azure-native:location westus2
```

3. Stand up the cluster by invoking pulumi
```bash
$ pulumi up
```

4. Navigate to https://web.azuresynapse.net and sign in to your new workspace.

5. From there, feel free to experiment. Simply making edits and running `pulumi up` will incrementally update your
stack.

6. Once you've finished experimenting, tear down your stack's resources by destroying and removing it:

```bash
$ pulumi destroy --yes
$ pulumi stack rm --yes
```
6 changes: 6 additions & 0 deletions examples/azure-synapse/project.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
//> using scala "3.3.1"
//> using options -Werror -Wunused:all -Wvalue-discard -Wnonunit-statement
//> using dep "org.virtuslab::besom-core:0.4.0-SNAPSHOT"
//> using dep "org.virtuslab::besom-random:4.16.1-core.0.4-SNAPSHOT"
//> using dep "org.virtuslab::besom-azure-native:2.38.0-core.0.4-SNAPSHOT"
//> using repository sonatype:snapshots
Loading