From 05e73860c9a9e4eaea49838f2b32386b032dca96 Mon Sep 17 00:00:00 2001 From: Connor Bernard Date: Sat, 12 Oct 2024 08:24:27 +0000 Subject: [PATCH 1/5] feat: remove error function in favor of log level fatal --- scripts/github-repos/github-repos.rb | 53 ++++++++++++++-------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/scripts/github-repos/github-repos.rb b/scripts/github-repos/github-repos.rb index 0ec670d..0ae3cf4 100755 --- a/scripts/github-repos/github-repos.rb +++ b/scripts/github-repos/github-repos.rb @@ -8,10 +8,11 @@ def main() puts "Script start." org = OrgManager.new $opts = OptionParser.new do |opt| - opt.banner = "Usage: #{__FILE__} [required options] [invite|repos|remove] - GITHUB_ORG_API_KEY for the org must be set as an environment variable. - 'invite' invites students provided in .csv file and creates teams, - 'repos' creates team repos, 'remove' remove students, repos, teams from the org." + opt.banner = "Usage: #{__FILE__} [required options] [invite|team_repos|individual_repos|remove|remove_access] + GITHUB_ORG_API_KEY for the org must be set as an environment variable. + 'invite' invites students provided in .csv file and creates teams, + 'team_repos' creates team repos, 'individual_repos' creates individual repos, + 'remove' remove students, repos, teams from the org." opt.on('-cCSVFILE', '--csv=CSVFILE', 'CSV file containing at least "Team" and "Email" named columns') do |csv| org.read_teams_and_emails_from csv end @@ -41,7 +42,9 @@ def main() when 'repos' then org.create_repos when 'remove' then org.remove when 'remove_access' then org.remove_access - else org.print_error + else + STDERR.puts $opts + exit 1 end puts "Run successfully." puts "Script ends." @@ -55,8 +58,8 @@ def initialize @base_filename = nil @semester = nil @template = nil - @childteams = Hash.new { |hash, key| hash[key] = [] } # teamID => [email1, email2, ...] - print_error("GITHUB_ORG_API_KEY not defined in environment") unless (@key = ENV['GITHUB_ORG_API_KEY']) + @childteams = Hash.new { |hash, key| hash[key] = [] } + log("GITHUB_ORG_API_KEY not defined in environment", :fatal) unless (@key = ENV['GITHUB_ORG_API_KEY']) @client = Octokit::Client.new(access_token: @key) end @@ -80,21 +83,16 @@ def valid? end def log(msg, type=:info, output_file=nil) - output_file ||= STDERR if type === :error + output_file ||= STDERR if type === :error || type === :fatal output_file ||= STDOUT output_file.puts "[#{type.upcase}]: #{msg}" - end - - def print_error(msg=nil) - log(msg, :error) if !msg.nil? - STDERR.puts $opts - exit 1 + exit 1 if type === :fatal end def read_teams_and_emails_from csv data = CSV.parse(IO.read(csv), headers: true) hash = data.first.to_h - print_error "Need at least 'Team' (int) and 'Email' (str) columns in #{csv}" unless + log("Need at least 'Team' (int) and 'Email' (str) columns in #{csv}", :fatal) unless hash.has_key?('Team') && hash.has_key?('Email') log "geting GitHub users. Please wait..." data.each do |row| @@ -107,7 +105,7 @@ def read_teams_and_emails_from csv user['uid'] = @client.user(username).id rescue Octokit::NotFound user['username'] = nil - log("GitHub Account '#{username}' does not exist. Using '#{row['email']}' instead") + log("GitHub Account '#{username}' does not exist. Using '#{row['Email']}' instead") end else log "no gh username for user #{row['Email']}; using email instead" @@ -118,7 +116,7 @@ def read_teams_and_emails_from csv end def invite - print_error "csv file, base filename, template repo name, semester prefix, students team name, and gsi team name needed." unless valid? + log("csv file, base filename, template repo name, semester prefix, students team name, and gsi team name needed.", :fatal) unless valid? first_child_team_name = %Q{#{@semester}-#{@childteams.keys[0]}} begin @@ -190,28 +188,31 @@ def invite end end - def create_repos - print_error "csv file, base filename, template repo name, semester prefix, students team name, and gsi team name needed." unless valid? + def create_team_repos + log("csv file, base filename, template repo name, semester prefix, students team name, and gsi team name needed.", :fatal) unless valid? @childteams.each_key do |team| begin team_id = @client.team_by_name(@orgname, %Q{#{@semester}-#{team}})['id'] rescue Octokit::NotFound - print_error "students teams information mismatched - could not find team '#{@semester}-#{team}' in org '#{@orgname}'" + log("students teams information mismatched - could not find team '#{@semester}-#{team}' in org '#{@orgname}'", :fatal) end gsiteam_id = @client.team_by_name(@orgname, @gsiteam)['id'] new_repo_name = %Q{#{@semester}-#{@base_filename}-#{team}} if !@client.repository? %Q{#{@orgname}/#{new_repo_name}} begin - new_repo = @client.create_repository_from_template(%Q{#{@orgname}/#{@template}}, new_repo_name, - {owner: @orgname, private: true}) + new_repo = @client.create_repository_from_template( + @template, + new_repo_name, + {owner: @orgname, private: true}, + ) log "created repo '#{new_repo_name}' from template '#{@template}' in org '#{@orgname}'" rescue Octokit::NotFound - print_error "failed to create repo: template not found." + log("failed to create repo: template not found.", :fatal) end if @client.add_team_repository(team_id, new_repo['full_name'], {permission: 'push'}) log "added repo '#{new_repo_name}' to team '#{@semester}-#{team}' with permission 'push' in org '#{@orgname}'" else - log("failed to add repo '#{new_repo_name}' to team '#{@semester}-#{team}' with permission 'push' in org '#{@orgname}'", :warn) + log("failed to add repo '#{new_repo_name}' to team '#{@semester}-#{team}' with permission 'push' in org '#{@orgname}'", :error) end if @client.add_team_repository(gsiteam_id, new_repo['full_name'], {permission: 'admin'}) log "added repo '#{new_repo_name}' to team '#{@gsiteam}' with permission 'admin' in org '#{@orgname}'" @@ -223,7 +224,7 @@ def create_repos end def remove - print_error "csv file, base filename, template repo name, semester prefix, students team name, and gsi team name needed." unless valid? + log("csv file, base filename, template repo name, semester prefix, students team name, and gsi team name needed.", :fatal) unless valid? # remove and delete all repos from the students team, delete all child teams # also cancel all pending invitaions @childteams.each_key do |team| @@ -231,7 +232,7 @@ def remove if @client.delete_repository(repo_name) log "deleted repo '#{@semester}-#{@base_filename}-#{team}' from org #{@orgname}" else - log("failed to delete repo '#{@semester}-#{@base_filename}-#{team}' from org '#{@orgname}'", :warn) + log("failed to delete repo '#{@semester}-#{@base_filename}-#{team}' from org '#{@orgname}'", :error) end begin childteam_id = @client.team_by_name(@orgname, %Q{#{@semester}-#{team}})['id'] # eg slug fa23-01 From e5e1da13716a71bbb39037e4b5d3d900ff732993 Mon Sep 17 00:00:00 2001 From: Connor Bernard Date: Sat, 12 Oct 2024 08:25:24 +0000 Subject: [PATCH 2/5] feat: add support for creating individual repos --- scripts/github-repos/github-repos.rb | 51 +++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/scripts/github-repos/github-repos.rb b/scripts/github-repos/github-repos.rb index 0ae3cf4..db62e07 100755 --- a/scripts/github-repos/github-repos.rb +++ b/scripts/github-repos/github-repos.rb @@ -39,7 +39,8 @@ def main() command = ARGV.pop case command when 'invite' then org.invite - when 'repos' then org.create_repos + when 'team_repos' then org.create_team_repos + when 'individual_repos' then org.create_individual_repos when 'remove' then org.remove when 'remove_access' then org.remove_access else @@ -223,6 +224,54 @@ def create_team_repos end end + def create_individual_repos + log("csv file, base filename, template repo name, semester prefix, students team name, and gsi team name needed.", :fatal) unless valid? + gsiteam_id = @client.team_by_name(@orgname, @gsiteam)['id'] + users = @childteams.values.flatten + log("all users must have GitHub usernames to create individual repos", :fatal) unless users.all? { |user| user['username'] } + did_fail_to_add_all_users = false + users.each do |user| + # their email username after replacing non-alphanumeric chars with '-' + email_username_sanitized = user['email'][/^.*(?=@)/].gsub(/\W|_/, '-') + curr_repo_name = "#{@semester}-#{email_username_sanitized}-#{@base_filename}" + curr_repo = nil + begin + curr_repo = @client.repository "#{@orgname}/#{curr_repo_name}" + rescue Octokit::NotFound + begin + curr_repo = @client.create_repository_from_template( + @template, + curr_repo_name, + {owner: @orgname, private: true}, + ) + if curr_repo + log "created repo '#{curr_repo_name}' from template '#{@template}' in org '#{@orgname}'" + else + log("failed to creat repo '#{curr_repo_name}' from template '#{@template}' in org '#{@orgname}'", :error) + next + end + rescue Octokit::NotFound + log("failed to create repo: template not found.", :fatal) + end + end + if @client.add_team_repository(gsiteam_id, curr_repo['full_name'], {permission: 'admin'}) + log "added repo '#{curr_repo_name}' to team '#{@gsiteam}' with permission 'admin' in org '#{@orgname}'" + else + log("failed to add repo '#{curr_repo_name}' to team '#{@gsiteam}' with permission 'admin' in org '#{@orgname}'", :warn) + end + begin + @client.invite_user_to_repository(curr_repo['full_name'], user['username']) + log "invited user '#{user['username']}' to repo '#{curr_repo['full_name']}' in org '#{@orgname}'" + rescue Octokit::Forbidden + did_fail_to_add_all_users = true + log("Could find GitHub user '#{user['username']}' in org '#{@orgname}' to add to repo '#{curr_repo_name}'", :error) + end + end + log("Could not add all users. See error logs", :fatal) if did_fail_to_add_all_users + puts @client.say "Let the CHIPs begin" + return + end + def remove log("csv file, base filename, template repo name, semester prefix, students team name, and gsi team name needed.", :fatal) unless valid? # remove and delete all repos from the students team, delete all child teams From f17566a80c9a63af7f2bd8cf63d2d8bf8cb84069 Mon Sep 17 00:00:00 2001 From: Connor Bernard Date: Sat, 12 Oct 2024 08:25:47 +0000 Subject: [PATCH 3/5] docs: update readme with new docs --- scripts/github-repos/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/github-repos/README.md b/scripts/github-repos/README.md index 413110f..9c7cc66 100644 --- a/scripts/github-repos/README.md +++ b/scripts/github-repos/README.md @@ -1,11 +1,11 @@ # Bulk creation/deletion of many repos and cs169a-team ```text -Usage: ./github-repos.rb [required options] [invite|repos|remove|remove_access] +Usage: ./github-repos.rb [required options] [invite|team_repos|remove|remove_access] GITHUB_ORG_API_KEY for the org must be set as an environment variable. -'invite' invites students provided in .csv file and creates teams, 'repos' creates team repos, 'remove' remove students, repos, teams from the org +'invite' invites students provided in .csv file and creates teams, 'team_repos' creates team repos, 'remove' remove students, repos, teams from the org It's safe to run multiple times. @@ -14,7 +14,7 @@ Required arguments: -o, --orgname=ORGNAME The name of the org eg org_name -f, --filename=FILENAME The base filename for repos, eg "fa23-actionmap-04", actionmap is the base file name of the repo -p, --prefix=PREFIX Semester prefix, eg "fa23" create a repos prefix, "fa23-actionmap-04", etc. - -t, --template=TEMPLATE The repo name within the org to use as template eg repo_name (Assume the repo own by org) + -t, --template=TEMPLATE The repo template to use to generate individual or team repos (should be of format org/repo-name), eg saasbook/chips-3.5 -s, --studentteam=STUDENTTEAM The team name of all the students team -g, --gsiteam=GSITEAM The team name of all the staff team ``` From 535d5bf95a4a1ecafc10b5211fbf81eb4b1ef2ea Mon Sep 17 00:00:00 2001 From: Connor Bernard Date: Sat, 12 Oct 2024 08:27:40 +0000 Subject: [PATCH 4/5] docs(example): add example csv --- scripts/github-repos/example_sheet.csv | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 scripts/github-repos/example_sheet.csv diff --git a/scripts/github-repos/example_sheet.csv b/scripts/github-repos/example_sheet.csv new file mode 100644 index 0000000..dd36912 --- /dev/null +++ b/scripts/github-repos/example_sheet.csv @@ -0,0 +1,2 @@ +Team,Email,Name,What is your student ID?,GitHub Username +16,connorbernard@berkeley.edu,Connor Bernard,3035597811,connor-bernard From 2c4334c8ef33e053f9ed348650d06dae37d9be07 Mon Sep 17 00:00:00 2001 From: Connor Bernard Date: Sat, 12 Oct 2024 09:12:37 +0000 Subject: [PATCH 5/5] fix: handle github 422 when they should 429 --- scripts/github-repos/github-repos.rb | 36 ++++++++++++++++------------ 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/scripts/github-repos/github-repos.rb b/scripts/github-repos/github-repos.rb index db62e07..ebc9d6f 100755 --- a/scripts/github-repos/github-repos.rb +++ b/scripts/github-repos/github-repos.rb @@ -105,8 +105,7 @@ def read_teams_and_emails_from csv begin user['uid'] = @client.user(username).id rescue Octokit::NotFound - user['username'] = nil - log("GitHub Account '#{username}' does not exist. Using '#{row['Email']}' instead") + log("GitHub Account '#{username}' does not exist. Using '#{row['Email']}' instead", :warn) end else log "no gh username for user #{row['Email']}; using email instead" @@ -238,20 +237,27 @@ def create_individual_repos begin curr_repo = @client.repository "#{@orgname}/#{curr_repo_name}" rescue Octokit::NotFound - begin - curr_repo = @client.create_repository_from_template( - @template, - curr_repo_name, - {owner: @orgname, private: true}, - ) - if curr_repo - log "created repo '#{curr_repo_name}' from template '#{@template}' in org '#{@orgname}'" - else - log("failed to creat repo '#{curr_repo_name}' from template '#{@template}' in org '#{@orgname}'", :error) - next + if !curr_repo + begin + curr_repo ||= @client.create_repository_from_template( + @template, + curr_repo_name, + {owner: @orgname, private: true}, + ) + if curr_repo + log "created repo '#{curr_repo_name}' from template '#{@template}' in org '#{@orgname}'" + else + log("failed to create repo '#{curr_repo_name}' from template '#{@template}' in org '#{@orgname}'", :error) + next + end + rescue Octokit::NotFound + log("failed to create repo: template not found.", :fatal) + # apparently they don't know what 429 errors are, so they just 422 instead? + rescue Octokit::UnprocessableEntity + log("rate limited. The script will resume in one minute", :warn) + sleep 60 + retry end - rescue Octokit::NotFound - log("failed to create repo: template not found.", :fatal) end end if @client.add_team_repository(gsiteam_id, curr_repo['full_name'], {permission: 'admin'})