Skip to content

Commit

Permalink
feat: add job to overwrite/update all video metadata in s3 weekly
Browse files Browse the repository at this point in the history
  • Loading branch information
PThorpe92 committed Dec 16, 2024
1 parent 5875381 commit d4ce441
Show file tree
Hide file tree
Showing 12 changed files with 165 additions and 31 deletions.
9 changes: 5 additions & 4 deletions .air.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,13 @@ root = "."
tmp_dir = "bin"

[build]
cmd = "go build -o ./bin/main backend/cmd/main.go"
bin = "bin/main"
include_dir = ["backend"]
cmd = "go build -o ./bin/main /app/backend/cmd/main.go"
bin = "./bin/main"
include_dir = ["./backend"]
include_file = []
exclude_file = []
exclude_regex = ["_test\\.go"]
exclude_ext = [".tsx", ".ts", ".json"]
exclude_regex = ["_test\\.go", "frontend/.*", "provider-middleware/.*"]
exclude_unchanged = true
# Follow symlink for directories
follow_symlink = true
Expand Down
10 changes: 5 additions & 5 deletions .middleware.air.toml
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
root = "."
tmp_dir = "tmp"
tmp_dir = "bin"

[build]
cmd = "go build -o ./bin/provider-middleware ./provider-middleware/."
bin = "bin/provider-middleware"
include_dir = ["backend", "provider-middleware"]
cmd = "go build -o ./bin/provider-middleware /app/provider-middleware/."
bin = "./bin/provider-middleware"
include_dir = ["./backend", "./provider-middleware"]
include_file = []
exclude_dir = ["config", "backend/tasks"]
exclude_dir = ["./config", "./backend/tasks"]
exclude_regex = ["_test\\.go"]
exclude_unchanged = true

Expand Down
8 changes: 4 additions & 4 deletions .tasks.air.toml
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
root = "."
tmp_dir = "tmp"
tmp_dir = "bin"

[build]
cmd = "go build -o ./bin/cron_tasks ./backend/tasks/."
cmd = "go build -o ./bin/cron_tasks /app/backend/tasks/."
bin = "bin/cron_tasks"
include_dir = ["backend"]
include_dir = ["backend", "backend/tasks"]
include_file = []
exclude_dir = ["frontend", "provider-middleware"]
exclude_regex = ["_test\\.go"]
exclude_regex = ["_test\\.go", "frontend/.*", "provider-middleware/.*"]
exclude_unchanged = true

log = "logs/tasks.air.log"
Expand Down
2 changes: 1 addition & 1 deletion backend/dev.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ FROM golang:1.23-alpine

WORKDIR /app

RUN go install github.com/air-verse/air@latest
RUN go install github.com/air-verse/air@v1.61.0

COPY backend/go.mod backend/go.sum ./
RUN go mod download
Expand Down
25 changes: 14 additions & 11 deletions backend/src/models/jobs.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ func (cj *CronJob) BeforeCreate(tx *gorm.DB) error {
schedule = EveryThreeHours
}
cj.Schedule = schedule
case string(PutVideoMetadataJob):
cj.Schedule = EverySundayAt8PM
default:
cj.Schedule = os.Getenv("MIDDLEWARE_CRON_SCHEDULE")
}
Expand Down Expand Up @@ -66,19 +68,20 @@ const (
GetCoursesJob JobType = "get_courses"
GetActivityJob JobType = "get_activity"

ScrapeKiwixJob JobType = "scrape_kiwix"
RetryVideoDownloadsJob JobType = "retry_video_downloads"
RetryManualDownloadJob JobType = "retry_manual_download"
SyncVideoMetadataJob JobType = "sync_video_metadata"
AddVideosJob JobType = "add_videos"
EveryThreeHours string = "0 */3 * * *"

StatusPending JobStatus = "pending"
StatusRunning JobStatus = "running"
ScrapeKiwixJob JobType = "scrape_kiwix"
RetryVideoDownloadsJob JobType = "retry_video_downloads"
RetryManualDownloadJob JobType = "retry_manual_download"
SyncVideoMetadataJob JobType = "sync_video_metadata"
PutVideoMetadataJob JobType = "put_video_metadata"
AddVideosJob JobType = "add_videos"
EveryThreeHours string = "0 */3 * * *"
EverySundayAt8PM string = "0 20 * * 7"
StatusPending JobStatus = "pending"
StatusRunning JobStatus = "running"
)

var AllDefaultProviderJobs = []JobType{GetCoursesJob, GetMilestonesJob, GetActivityJob}
var AllContentProviderJobs = []JobType{ScrapeKiwixJob, RetryVideoDownloadsJob, SyncVideoMetadataJob}
var AllContentProviderJobs = []JobType{ScrapeKiwixJob, RetryVideoDownloadsJob, SyncVideoMetadataJob, PutVideoMetadataJob}

func (jt JobType) PubName() string {
return fmt.Sprintf("tasks.%s", string(jt))
Expand All @@ -87,7 +90,7 @@ func (jt JobType) PubName() string {
func (jt JobType) GetParams(db *gorm.DB, provId uint, jobId string) (map[string]interface{}, error) {
var skip bool
switch jt {
case RetryVideoDownloadsJob, SyncVideoMetadataJob, ScrapeKiwixJob:
case RetryVideoDownloadsJob, SyncVideoMetadataJob, PutVideoMetadataJob, ScrapeKiwixJob:
return map[string]interface{}{
"job_id": jobId,
"job_type": jt,
Expand Down
5 changes: 4 additions & 1 deletion backend/tasks/dev.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,11 @@ FROM golang:1.23-alpine

WORKDIR /app

RUN go install github.com/air-verse/air@latest
RUN mkdir -p /app/tmp /app/bin
RUN go install github.com/air-verse/[email protected]

COPY backend/tasks/ /app/backend/tasks/
COPY backend/tasks/go.mod backend/tasks/go.sum ./
RUN go mod download

CMD ["air", "-c", ".tasks.air.toml"]
2 changes: 2 additions & 0 deletions backend/tasks/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,8 @@ func initLogging() {
func getCronSchedule(task *models.RunnableTask, hour int) string {
if task.Provider != nil && task.Provider.Type == models.Brightspace {
return fmt.Sprintf("0 0 %d * * 4", hour)
} else if task.Job.Name == string(models.PutVideoMetadataJob) {
return models.EverySundayAt8PM
} else {
return task.Job.Schedule
}
Expand Down
2 changes: 1 addition & 1 deletion backend/tasks/scheduler.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ func (jr *JobRunner) generateOpenContentProviderTasks() ([]models.RunnableTask,
}
otherTasks = append(otherTasks, *task)
}
case models.RetryVideoDownloadsJob, models.SyncVideoMetadataJob:
case models.RetryVideoDownloadsJob, models.PutVideoMetadataJob, models.SyncVideoMetadataJob:
job := models.CronJob{Name: string(jobType)}
if providers[idx].Title == models.Youtube {
task, err := jr.handleCreateOCProviderTask(&job, providers[idx].ID)
Expand Down
Loading

0 comments on commit d4ce441

Please sign in to comment.