Skip to content

Commit

Permalink
Qualifying all JLD2.save and JLD2.load functions explicitly.
Browse files Browse the repository at this point in the history
  • Loading branch information
codetalker7 committed Jul 25, 2024
1 parent 0673388 commit 4270b71
Show file tree
Hide file tree
Showing 3 changed files with 13 additions and 13 deletions.
2 changes: 1 addition & 1 deletion src/indexing/codecs/residual.jl
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,6 @@ A vector of codes for the specified chunk.
"""
function load_codes(codec::ResidualCodec, chunk_idx::Int)
codes_path = joinpath(codec.config.indexing_settings.index_path, "$(chunk_idx).codes.jld2")
codes = load(codes_path, "codes")
codes = JLD2.load(codes_path, "codes")
codes
end
6 changes: 3 additions & 3 deletions src/indexing/collection_indexer.jl
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ function _sample_embeddings(indexer::CollectionIndexer, sampled_pids::Set{Int})
sample_path = joinpath(indexer.config.indexing_settings.index_path, "sample.jld2")
@info "avg_doclen_est = $(indexer.avg_doclen_est) \t length(local_sample) = $(length(local_sample))"
@info "Saving sampled embeddings to $(sample_path)."
save(sample_path, Dict("local_sample_embs" => local_sample_embs))
JLD2.save(sample_path, Dict("local_sample_embs" => local_sample_embs))

indexer.avg_doclen_est
end
Expand Down Expand Up @@ -182,7 +182,7 @@ The tuple `sample, sample_heldout`.
function _concatenate_and_split_sample(indexer::CollectionIndexer)
# load the sample embeddings
sample_path = joinpath(indexer.config.indexing_settings.index_path, "sample.jld2")
sample = load(sample_path, "local_sample_embs")
sample = JLD2.load(sample_path, "local_sample_embs")
@debug "Original sample shape: $(size(sample))"

# randomly shuffle embeddings
Expand Down Expand Up @@ -357,7 +357,7 @@ function _build_ivf(indexer::CollectionIndexer)

@info "Saving the IVF."
ivf_path = joinpath(indexer.config.indexing_settings.index_path, "ivf.jld2")
save(ivf_path, Dict(
JLD2.save(ivf_path, Dict(
"ivf" => ivf,
"ivf_lengths" => ivf_lengths
))
Expand Down
18 changes: 9 additions & 9 deletions src/indexing/index_saver.jl
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@ The path of of the codec is inferred from the config stored in `saver`.
"""
function load_codec!(saver::IndexSaver)
index_path = saver.config.indexing_settings.index_path
centroids = load(joinpath(index_path, "centroids.jld2"), "centroids")
avg_residual = load(joinpath(index_path, "avg_residual.jld2"), "avg_residual")
buckets = load(joinpath(index_path, "buckets.jld2"))
centroids = JLD2.load(joinpath(index_path, "centroids.jld2"), "centroids")
avg_residual = JLD2.load(joinpath(index_path, "avg_residual.jld2"), "avg_residual")
buckets = JLD2.load(joinpath(index_path, "buckets.jld2"))
saver.codec = ResidualCodec(saver.config, centroids, avg_residual, buckets["bucket_cutoffs"], buckets["bucket_weights"])
end

Expand All @@ -57,9 +57,9 @@ function save_codec(saver::IndexSaver)
buckets_path = joinpath(index_path, "buckets.jld2")
@info "Saving codec to $(centroids_path), $(avg_residual_path) and $(buckets_path)"

save(centroids_path, Dict("centroids" => saver.codec.centroids))
save(avg_residual_path, Dict("avg_residual" => saver.codec.avg_residual))
save(
JLD2.save(centroids_path, Dict("centroids" => saver.codec.centroids))
JLD2.save(avg_residual_path, Dict("avg_residual" => saver.codec.avg_residual))
JLD2.save(
buckets_path,
Dict(
"bucket_cutoffs" => saver.codec.bucket_cutoffs,
Expand Down Expand Up @@ -92,13 +92,13 @@ function save_chunk(saver::IndexSaver, chunk_idx::Int, offset::Int, embs::Matrix
codes_path = "$(path_prefix).codes.jld2"
residuals_path = "$(path_prefix).residuals.jld2"
@info "Saving compressed codes to $(codes_path) and residuals to $(residuals_path)"
save(codes_path, Dict("codes" => codes))
save(residuals_path, Dict("residuals" => residuals))
JLD2.save(codes_path, Dict("codes" => codes))
JLD2.save(residuals_path, Dict("residuals" => residuals))

# saving doclens
doclens_path = joinpath(saver.config.indexing_settings.index_path, "doclens.$(chunk_idx).jld2")
@info "Saving doclens to $(doclens_path)"
save(doclens_path, Dict("doclens" => doclens))
JLD2.save(doclens_path, Dict("doclens" => doclens))

# the metadata
metadata_path = joinpath(saver.config.indexing_settings.index_path, "$(chunk_idx).metadata.json")
Expand Down

0 comments on commit 4270b71

Please sign in to comment.