Sourcery refactored tuning branch

modules/silicon_design/scripts/build/images/provision/papermill-launcher

@@ -21,8 +21,5 @@ def expand_path(p):
 _, input_path, output_path, *params_args = sys.argv
 input_path = expand_path(input_path)
 output_path = expand_path(output_path)
-parameters = dict(
-    (k, expand_path(v))
-    for k, v in args(params_args)
-)
+parameters = {k: expand_path(v) for k, v in args(params_args)}
 pm.execute_notebook(input_path, output_path, parameters=parameters, progress_bar=False)

radlab-launcher/radlab.py

@@ -89,7 +89,9 @@ def main(varcontents={}, module_name=None, action=None, projid=None, tfbucket=No
 def radlabauth(currentusr):
     try:
         token = subprocess.Popen(["gcloud auth application-default print-access-token"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).stdout.read().strip().decode('utf-8')
-        r = requests.get('https://www.googleapis.com/oauth2/v3/tokeninfo?access_token=' + token)
+        r = requests.get(
+            f'https://www.googleapis.com/oauth2/v3/tokeninfo?access_token={token}'
+        )
         currentusr = r.json()["email"]
 
         # Setting Credentials for non Cloud Shell CLI
@@ -126,9 +128,11 @@ def radlabauth(currentusr):
             sys.exit(Fore.RED + "\nError Occured - INVALID choice.\n")
         else:
             token = subprocess.Popen(["gcloud auth application-default print-access-token"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).stdout.read().strip().decode('utf-8')
-            r = requests.get('https://www.googleapis.com/oauth2/v3/tokeninfo?access_token=' + token)
+            r = requests.get(
+                f'https://www.googleapis.com/oauth2/v3/tokeninfo?access_token={token}'
+            )
             currentusr = r.json()["email"]
-            os.system("gcloud config set account " + currentusr)
+            os.system(f"gcloud config set account {currentusr}")
             print(
                 "\nUser to deploy RAD Lab Modules (Selected) : " + Fore.GREEN + Style.BRIGHT + currentusr + Style.RESET_ALL)
             return currentusr
@@ -141,15 +145,12 @@ def set_proj(projid):
             select_proj = input("\nWhich Project would you like to use for RAD Lab management (Example - Creating/Utilizing GCS bucket where Terraform states will be stored) ? :" + "\n[1] Currently set project - " + Fore.GREEN + projid + Style.RESET_ALL + "\n[2] Enter a different Project ID" + Fore.YELLOW + Style.BRIGHT + "\nChoose a number for the RAD Lab management Project" + Style.RESET_ALL + ': ').strip()
             if (select_proj == '2'):
                 projid = input(Fore.YELLOW + Style.BRIGHT + "Enter the Project ID" + Style.RESET_ALL + ': ').strip()
-            elif (select_proj != '1' and select_proj != '2'):
+            elif select_proj != '1':
                 sys.exit(Fore.RED + "\nError Occured - INVALID choice.\n")
         else:
             projid = input(Fore.YELLOW + Style.BRIGHT + "\nEnter the Project ID for RAD Lab management" + Style.RESET_ALL + ': ').strip()
-    else:
-        pass
-
-    os.system("gcloud config set project " + projid)
-    os.system("gcloud auth application-default set-quota-project "  + projid )
+    os.system(f"gcloud config set project {projid}")
+    os.system(f"gcloud auth application-default set-quota-project {projid}")
     print("\nProject ID (Selected) : " + Fore.GREEN + Style.BRIGHT + projid + Style.RESET_ALL)
 
     return projid
@@ -158,13 +159,10 @@ def set_proj(projid):
 def launcherperm(projid, currentusr):
     # Hardcoded Project level required RAD Lab Launcher roles
     launcherprojroles = ['roles/storage.admin', 'roles/serviceusage.serviceUsageConsumer']
-    # Hardcoded Org level required RAD Lab Launcher roles
-    launcherorgroles = ['roles/iam.organizationRoleViewer']
-
     credentials = GoogleCredentials.get_application_default()
 
     service0 = discovery.build('cloudresourcemanager', 'v3', credentials=credentials)
-    request0 = service0.projects().getIamPolicy(resource='projects/' + projid)
+    request0 = service0.projects().getIamPolicy(resource=f'projects/{projid}')
     response0 = request0.execute()
 
     projiam = True
@@ -178,21 +176,23 @@ def launcherperm(projid, currentusr):
             # print(response0['bindings'][y]['members'])
 
             # Check for Owner role on RAD Lab Management Project
-            if (response0['bindings'][y]['role'] == 'roles/owner' and 'user:' + currentusr in response0['bindings'][y]['members']):
+            if (
+                response0['bindings'][y]['role'] == 'roles/owner'
+                and f'user:{currentusr}' in response0['bindings'][y]['members']
+            ):
                 rolefound = True
                 ownerrole = True
                 print("\n" + currentusr + " has roles/owner role for RAD Lab Management Project: " + projid)
                 break
-            # Check for Required roles on RAD Lab Management Project
             elif (response0['bindings'][y]['role'] == role):
                 rolefound = True
-                if ('user:' + currentusr not in response0['bindings'][y]['members']):
+                if (
+                    f'user:{currentusr}'
+                    not in response0['bindings'][y]['members']
+                ):
                     projiam = False
                     sys.exit(
                         Fore.RED + "\nError Occured - RADLAB LAUNCHER PERMISSION ISSUE | " + role + " permission missing...\n(Review https://github.com/GoogleCloudPlatform/rad-lab/tree/main/radlab-launcher#iam-permissions-prerequisites for more details)\n" + Style.RESET_ALL)
-                else:
-                    pass
-
         if rolefound == False:
             sys.exit(
                 Fore.RED + "\nError Occured - RADLAB LAUNCHER PERMISSION ISSUE | " + role + " permission missing...\n(Review https://github.com/GoogleCloudPlatform/rad-lab/tree/main/radlab-launcher#iam-permissions-prerequisites for more details)\n" + Style.RESET_ALL)
@@ -204,7 +204,7 @@ def launcherperm(projid, currentusr):
         print(Fore.GREEN + '\nRADLAB LAUNCHER - Project Permission check passed' + Style.RESET_ALL)
 
     service1 = discovery.build('cloudresourcemanager', 'v3', credentials=credentials)
-    request1 = service1.projects().get(name='projects/' + projid)
+    request1 = service1.projects().get(name=f'projects/{projid}')
     response1 = request1.execute()
 
     if 'parent' in response1.keys():
@@ -214,6 +214,9 @@ def launcherperm(projid, currentusr):
         response2 = request2.execute()
 
         orgiam = True
+        # Hardcoded Org level required RAD Lab Launcher roles
+        launcherorgroles = ['roles/iam.organizationRoleViewer']
+
         for role in launcherorgroles:
             rolefound = False
             for x in range(len(response2['bindings'])):
@@ -223,12 +226,12 @@ def launcherperm(projid, currentusr):
                 # print(response2['bindings'][x]['members'])
                 if (role == response2['bindings'][x]['role']):
                     rolefound = True
-                    if ('user:' + currentusr not in response2['bindings'][x]['members']):
+                    if (
+                        f'user:{currentusr}'
+                        not in response2['bindings'][x]['members']
+                    ):
                         orgiam = False
                         sys.exit(Fore.RED + "\nError Occured - RADLAB LAUNCHER PERMISSION ISSUE | " + role + " permission missing...\n(Review https://github.com/GoogleCloudPlatform/rad-lab/tree/main/radlab-launcher#iam-permissions-prerequisites for more details)\n" + Style.RESET_ALL)
-                    else:
-                        pass
-
             if rolefound == False:
                 sys.exit(Fore.RED + "\nError Occured - RADLAB LAUNCHER PERMISSION ISSUE | " + role + " permission missing...\n(Review https://github.com/GoogleCloudPlatform/rad-lab/tree/main/radlab-launcher#iam-permissions-prerequisites for more details)\n" + Style.RESET_ALL)
 
@@ -239,15 +242,14 @@ def launcherperm(projid, currentusr):
 
 
 def findorg(parent):
-    if 'folders' in parent:
-        credentials = GoogleCredentials.get_application_default()
-        s = discovery.build('cloudresourcemanager', 'v3', credentials=credentials)
-        req = s.folders().get(name=parent)
-        res = req.execute()
-        return findorg(res['parent'])
-    else:
+    if 'folders' not in parent:
         # print(Fore.GREEN + "Org identified: " + Style.BRIGHT + parent + Style.RESET_ALL)
         return parent
+    credentials = GoogleCredentials.get_application_default()
+    s = discovery.build('cloudresourcemanager', 'v3', credentials=credentials)
+    req = s.folders().get(name=parent)
+    res = req.execute()
+    return findorg(res['parent'])
 
 
 def moduleperm(projid, module_name, currentusr):

radlab-launcher/terraform_installer.py

@@ -31,34 +31,28 @@ def main():
     reqs = requests.get(url)
     soup = BeautifulSoup(reqs.text, 'html.parser')
 
-    if('darwin' in system or 'linux' in system):
-        if('x86_64' in machine):
-            machine = 'amd64'
-        else:
-            machine = '386'
-
+    if ('darwin' in system or 'linux' in system):
+        machine = 'amd64' if ('x86_64' in machine) else '386'
         for link in soup.find_all('a'):
             # print(link.get('href'))
-            if("https://releases.hashicorp.com/terraform/" in str(link.get('href')) and ".zip" in str(link.get('href')) and system in str(link.get('href'))):
-
-                if(machine in str(link.get('href'))):
-                    downloadlink = link.get('href')
-                    # print(downloadlink)
-                    break
-        os.system("curl "+downloadlink+ " --output terraform_download.zip")
+            if (
+                "https://releases.hashicorp.com/terraform/"
+                in str(link.get('href'))
+                and ".zip" in str(link.get('href'))
+                and system in str(link.get('href'))
+            ) and (machine in str(link.get('href'))):
+                downloadlink = link.get('href')
+                # print(downloadlink)
+                break
+        os.system(f"curl {downloadlink} --output terraform_download.zip")
         os.system("unzip terraform_download.zip")
         print("\nPlease enter your machine's credentials to complete installation (if requested)...\n")
-        os.system("sudo mv " +os.getcwd()+"/terraform /usr/local/bin/")
+        os.system(f"sudo mv {os.getcwd()}/terraform /usr/local/bin/")
         os.remove("terraform_download.zip")
 
-    elif('windows' in system):
-        # Run Command Prompt as adminstrator
-
-        # Create installChocolatey.cmd
-        f=open('installChocolatey.cmd', 'w+')
-        f.write('@echo off\n\nSET DIR=%~dp0%\n\n::download install.ps1\n%systemroot%\System32\WindowsPowerShell\\v1.0\powershell.exe -NoProfile -ExecutionPolicy Bypass -Command "((new-object net.webclient).DownloadFile('"'https://community.chocolatey.org/install.ps1'"','"'%DIR%install.ps1'"'))"\n::run installer\n%systemroot%\System32\WindowsPowerShell\\v1.0\powershell.exe -NoProfile -ExecutionPolicy Bypass -Command "& '"'%DIR%install.ps1'"' %*"')
-        f.close()
-
+    elif ('windows' in system):
+        with open('installChocolatey.cmd', 'w+') as f:
+            f.write('@echo off\n\nSET DIR=%~dp0%\n\n::download install.ps1\n%systemroot%\System32\WindowsPowerShell\\v1.0\powershell.exe -NoProfile -ExecutionPolicy Bypass -Command "((new-object net.webclient).DownloadFile('"'https://community.chocolatey.org/install.ps1'"','"'%DIR%install.ps1'"'))"\n::run installer\n%systemroot%\System32\WindowsPowerShell\\v1.0\powershell.exe -NoProfile -ExecutionPolicy Bypass -Command "& '"'%DIR%install.ps1'"' %*"')
         # Install Chocolatey & Terraform
         os.system('installChocolatey.cmd')
         os.system('choco install terraform -y')

tools/build_readme.py

@@ -27,30 +27,30 @@ def main():
 
     # print(GITHUB_WORKSPACE)
 
-    modules_dir = GITHUB_WORKSPACE + '/modules'
+    modules_dir = f'{GITHUB_WORKSPACE}/modules'
 
-    for module in glob.glob(modules_dir + '/*'):
+    for module in glob.glob(f'{modules_dir}/*'):
 
         # print(module)
 
         try:
             # run the tfdoc.py
-            os.system('python3 tfdoc.py ' + module)
+            os.system(f'python3 tfdoc.py {module}')
 
         except Exception as e:
             raise SystemExit(e)
-    
+
     try: 
         # commit files
-        os.system('git config --local user.email ' + WORKFLOW_EMAIL)
-        os.system('git config --local user.name ' + WORKFLOW_USERNAME)
+        os.system(f'git config --local user.email {WORKFLOW_EMAIL}')
+        os.system(f'git config --local user.name {WORKFLOW_USERNAME}')
         os.system('git add -A')
         os.system('git commit -m "[WORKFLOW] Auto updating RAD-Lab Modules README.md" -a')
 
-        remote_repo="https://"+WORKFLOW_USERNAME+":"+WORKFLOW_PAT+"@github.com/"+GITHUB_REPOSITORY+".git"
+        remote_repo = f"https://{WORKFLOW_USERNAME}:{WORKFLOW_PAT}@github.com/{GITHUB_REPOSITORY}.git"
 
         # push changes
-        os.system('git push ' + remote_repo + ' HEAD:main --force')
+        os.system(f'git push {remote_repo} HEAD:main --force')
 
     except Exception as e:
         raise SystemExit(e)

tools/check-license.py

@@ -34,32 +34,31 @@ def main(PR):
 
         # Looping through all Open PRs
         for pr in response.json():
-            
+
             commentcheck = prcommentcheck(GITHUB_REPOSITORY, pr['number'])
             licensecheck(GITHUB_REPOSITORY, TOKEN, pr['number'],commentcheck)        
 
     else: 
-        print('Manual License check for: ' + PR)
+        print(f'Manual License check for: {PR}')
         licensecheck(GITHUB_REPOSITORY, TOKEN, int(PR),'false') 
 
 
 def open_pr(GITHUB_REPOSITORY):
     print('Fetching open PRs...')
     try:
-        response = requests.get('https://api.github.com/repos/'+ GITHUB_REPOSITORY +'/pulls')
-        return response
+        return requests.get(f'https://api.github.com/repos/{GITHUB_REPOSITORY}/pulls')
     except requests.exceptions.RequestException as e: 
         raise SystemExit(e)
 
 
 def licensecheck(GITHUB_REPOSITORY, TOKEN, pr, commentcheck):   
 
     # If commentcheck = 'false' i.e. License check has not run on the PR before.
-    if(commentcheck == 'false'):
+    if (commentcheck == 'false'):
     # if(checkmindiff(pr['created_at']) and commentcheck == 'false'):
 
-        print('PR # ' + str(pr) + ' : Run Licence check...')
-        
+        print(f'PR # {str(pr)} : Run Licence check...')
+
         # Get all pr files
         prfiles = pr_files(GITHUB_REPOSITORY,pr)
 
@@ -72,10 +71,10 @@ def licensecheck(GITHUB_REPOSITORY, TOKEN, pr, commentcheck):
         # print(glob.glob(os.getcwd()+'/temp/*/*/*'))
 
         # Run lisence check on the downloaded files in temp directory
-        pr_no_license_files = boilerplate(os.getcwd()+'/temp')
+        pr_no_license_files = boilerplate(f'{os.getcwd()}/temp')
 
         # Delete temp directory and its contents
-        shutil.rmtree(os.getcwd()+'/temp')
+        shutil.rmtree(f'{os.getcwd()}/temp')
 
         if pr_no_license_files:
             comment = '<!-- Boilerplate Check -->\nApache 2.0 License check failed!\n\nThe following files are missing the license boilerplate:\n'
@@ -89,14 +88,16 @@ def licensecheck(GITHUB_REPOSITORY, TOKEN, pr, commentcheck):
         commentpr(GITHUB_REPOSITORY, pr, comment, TOKEN)
 
     else:
-        print('PR # ' + str(pr) + ' : Skip Licence check...')
+        print(f'PR # {str(pr)} : Skip Licence check...')
 
 
 def prcommentcheck(GITHUB_REPOSITORY, pr):
     print('Checking if the License check has already ran...')
     try:
         status = 'false'
-        response = requests.get('https://api.github.com/repos/'+ GITHUB_REPOSITORY +'/issues/'+ str(pr) +'/comments')
+        response = requests.get(
+            f'https://api.github.com/repos/{GITHUB_REPOSITORY}/issues/{str(pr)}/comments'
+        )
         for comment in response.json():
             body = comment['body']
             if(body.startswith('<!-- Boilerplate Check -->')):
@@ -112,22 +113,24 @@ def boilerplate(local_temp):
     pr_no_license_files = []
     allfiles = check_boilerplate.main(local_temp)
     try:
-        for x in range(len(allfiles)):
-            pr_no_license_files.append(allfiles[x].replace(local_temp+'/', ""))
-        # print(pr_no_license_files)
+        pr_no_license_files.extend(
+            allfiles[x].replace(f'{local_temp}/', "")
+            for x in range(len(allfiles))
+        )
+            # print(pr_no_license_files)
     except:
         print("All files have the Apache 2.0 Lisence")
     return pr_no_license_files
 
 def pr_files(GITHUB_REPOSITORY,pr):
     pr_files = []
     try:
-        response = requests.get('https://api.github.com/repos/'+ GITHUB_REPOSITORY +'/pulls/'+ str(pr) +'/files')
-        for file in response.json():
-            if(file['status'] != 'removed'):
-                pr_files.append(file)
-            else:
-                continue
+        response = requests.get(
+            f'https://api.github.com/repos/{GITHUB_REPOSITORY}/pulls/{str(pr)}/files'
+        )
+        pr_files.extend(
+            file for file in response.json() if (file['status'] != 'removed')
+        )
         # print(pr_files)
         return pr_files
 
@@ -138,7 +141,7 @@ def downloadprfiles(prfiles):
     for file in prfiles:
         # print('Create Temp Directory')
         path = os.path.dirname(file['filename'])
-        path = os.getcwd() + '/temp/' + path
+        path = f'{os.getcwd()}/temp/{path}'
         # print(path)
 
         if not os.path.exists(path):
@@ -147,7 +150,7 @@ def downloadprfiles(prfiles):
         # print('Beginning file download with requests')
         r = requests.get(file['raw_url'])
 
-        with open(path + '/' + os.path.basename(file['filename']), 'wb') as f:
+        with open(f'{path}/' + os.path.basename(file['filename']), 'wb') as f:
             f.write(r.content)
 
         # # Retrieve HTTP meta-data
@@ -161,8 +164,12 @@ def commentpr(GITHUB_REPOSITORY, pr, comment, TOKEN):
     # print(comment)
     data = {"body":comment}
     try:
-        response  = requests.post('https://api.github.com/repos/'+ GITHUB_REPOSITORY +'/issues/'+ str(pr) +'/comments', data=json.dumps(data), headers=headers)
-        # print(response.text)
+        response = requests.post(
+            f'https://api.github.com/repos/{GITHUB_REPOSITORY}/issues/{str(pr)}/comments',
+            data=json.dumps(data),
+            headers=headers,
+        )
+            # print(response.text)
     except requests.exceptions.RequestException as e: 
         raise SystemExit(e)