Fix multipart uploads with checksums on object locked buckets #4118

Workflow file for this run

.github/workflows/object_store.yml at fe3d089

	# Licensed to the Apache Software Foundation (ASF) under one
	# or more contributor license agreements. See the NOTICE file
	# distributed with this work for additional information
	# regarding copyright ownership. The ASF licenses this file
	# to you under the Apache License, Version 2.0 (the
	# "License"); you may not use this file except in compliance
	# with the License. You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing,
	# software distributed under the License is distributed on an
	# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
	# KIND, either express or implied. See the License for the
	# specific language governing permissions and limitations
	# under the License.

	---
	# tests for `object_store` crate
	name: object_store

	concurrency:
	group: ${{ github.repository }}-${{ github.head_ref \|\| github.sha }}-${{ github.workflow }}
	cancel-in-progress: true

	# trigger for all PRs that touch certain files and changes to master
	on:
	push:
	branches:
	- master
	pull_request:
	paths:
	- object_store/**
	- .github/**

	jobs:
	clippy:
	name: Clippy
	runs-on: ubuntu-latest
	container:
	image: amd64/rust
	defaults:
	run:
	working-directory: object_store
	steps:
	- uses: actions/checkout@v4
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	- name: Setup Clippy
	run: rustup component add clippy
	# Run different tests for the library on its own as well as
	# all targets to ensure that it still works in the absence of
	# features that might be enabled by dev-dependencies of other
	# targets.
	- name: Run clippy with default features
	run: cargo clippy -- -D warnings
	- name: Run clippy with aws feature
	run: cargo clippy --features aws -- -D warnings
	- name: Run clippy with gcp feature
	run: cargo clippy --features gcp -- -D warnings
	- name: Run clippy with azure feature
	run: cargo clippy --features azure -- -D warnings
	- name: Run clippy with http feature
	run: cargo clippy --features http -- -D warnings
	- name: Run clippy with all features
	run: cargo clippy --all-features -- -D warnings
	- name: Run clippy with all features and all targets
	run: cargo clippy --all-features --all-targets -- -D warnings

	# test doc links still work
	#
	# Note that since object_store is not part of the main workspace,
	# this needs a separate docs job as it is not covered by
	# `cargo doc --workspace`
	docs:
	name: Rustdocs
	runs-on: ubuntu-latest
	defaults:
	run:
	working-directory: object_store
	env:
	RUSTDOCFLAGS: "-Dwarnings"
	steps:
	- uses: actions/checkout@v4
	- name: Run cargo doc
	run: cargo doc --document-private-items --no-deps --workspace --all-features

	# test the crate
	# This runs outside a container to workaround lack of support for passing arguments
	# to service containers - https://github.com/orgs/community/discussions/26688
	linux-test:
	name: Emulator Tests
	runs-on: ubuntu-latest
	defaults:
	run:
	working-directory: object_store
	env:
	# Disable full debug symbol generation to speed up CI build and keep memory down
	# "1" means line tables only, which is useful for panic tracebacks.
	RUSTFLAGS: "-C debuginfo=1"
	RUST_BACKTRACE: "1"
	# Run integration tests
	TEST_INTEGRATION: 1
	EC2_METADATA_ENDPOINT: http://localhost:1338
	AZURE_CONTAINER_NAME: test-bucket
	AZURE_STORAGE_USE_EMULATOR: "1"
	AZURITE_BLOB_STORAGE_URL: "http://localhost:10000"
	AZURITE_QUEUE_STORAGE_URL: "http://localhost:10001"
	AWS_BUCKET: test-bucket
	AWS_DEFAULT_REGION: "us-east-1"
	AWS_ACCESS_KEY_ID: test
	AWS_SECRET_ACCESS_KEY: test
	AWS_ENDPOINT: http://localhost:4566
	AWS_ALLOW_HTTP: true
	AWS_COPY_IF_NOT_EXISTS: dynamo:test-table:2000
	AWS_CONDITIONAL_PUT: dynamo:test-table:2000
	AWS_SERVER_SIDE_ENCRYPTION: aws:kms
	HTTP_URL: "http://localhost:8080"
	GOOGLE_BUCKET: test-bucket
	GOOGLE_SERVICE_ACCOUNT: "/tmp/gcs.json"

	steps:
	- uses: actions/checkout@v4

	# We are forced to use docker commands instead of service containers as we need to override the entrypoints
	# which is currently not supported - https://github.com/actions/runner/discussions/1872
	- name: Configure Fake GCS Server (GCP emulation)
	# Custom image - see fsouza/fake-gcs-server#1164
	run: \|
	echo "GCS_CONTAINER=$(docker run -d -p 4443:4443 tustvold/fake-gcs-server -scheme http -backend memory -public-host localhost:4443)" >> $GITHUB_ENV
	# Give the container a moment to start up prior to configuring it
	sleep 1
	curl -v -X POST --data-binary '{"name":"test-bucket"}' -H "Content-Type: application/json" "http://localhost:4443/storage/v1/b"
	echo '{"gcs_base_url": "http://localhost:4443", "disable_oauth": true, "client_email": "", "private_key": "", "private_key_id": ""}' > "$GOOGLE_SERVICE_ACCOUNT"

	- name: Setup WebDav
	run: docker run -d -p 8080:80 rclone/rclone serve webdav /data --addr :80

	- name: Setup LocalStack (AWS emulation)
	run: \|
	echo "LOCALSTACK_CONTAINER=$(docker run -d -p 4566:4566 localstack/localstack:3.8.1)" >> $GITHUB_ENV
	echo "EC2_METADATA_CONTAINER=$(docker run -d -p 1338:1338 amazon/amazon-ec2-metadata-mock:v1.9.2 --imdsv2)" >> $GITHUB_ENV
	aws --endpoint-url=http://localhost:4566 s3 mb s3://test-bucket
	aws --endpoint-url=http://localhost:4566 dynamodb create-table --table-name test-table --key-schema AttributeName=path,KeyType=HASH AttributeName=etag,KeyType=RANGE --attribute-definitions AttributeName=path,AttributeType=S AttributeName=etag,AttributeType=S --provisioned-throughput ReadCapacityUnits=5,WriteCapacityUnits=5

	KMS_KEY=$(aws --endpoint-url=http://localhost:4566 kms create-key --description "test key")
	echo "AWS_SSE_KMS_KEY_ID=$(echo $KMS_KEY \| jq -r .KeyMetadata.KeyId)" >> $GITHUB_ENV

	- name: Configure Azurite (Azure emulation)
	# the magical connection string is from
	# https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio#http-connection-strings
	run: \|
	echo "AZURITE_CONTAINER=$(docker run -d -p 10000:10000 -p 10001:10001 -p 10002:10002 mcr.microsoft.com/azure-storage/azurite)" >> $GITHUB_ENV
	az storage container create -n test-bucket --connection-string 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;QueueEndpoint=http://localhost:10001/devstoreaccount1;'

	- name: Setup Rust toolchain
	run: \|
	rustup toolchain install stable
	rustup default stable

	- name: Run object_store tests
	run: cargo test --features=aws,azure,gcp,http

	- name: Run object_store tests (AWS native conditional put)
	run: cargo test --features=aws
	env:
	AWS_CONDITIONAL_PUT: etag-put-if-not-exists
	AWS_COPY_IF_NOT_EXISTS: multipart

	- name: GCS Output
	if: ${{ !cancelled() }}
	run: docker logs $GCS_CONTAINER

	- name: LocalStack Output
	if: ${{ !cancelled() }}
	run: docker logs $LOCALSTACK_CONTAINER

	- name: EC2 Metadata Output
	if: ${{ !cancelled() }}
	run: docker logs $EC2_METADATA_CONTAINER

	- name: Azurite Output
	if: ${{ !cancelled() }}
	run: docker logs $AZURITE_CONTAINER

	# test the object_store crate builds against wasm32 in stable rust
	wasm32-build:
	name: Build wasm32
	runs-on: ubuntu-latest
	container:
	image: amd64/rust
	defaults:
	run:
	working-directory: object_store
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: true
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	with:
	target: wasm32-unknown-unknown,wasm32-wasi
	- name: Build wasm32-unknown-unknown
	run: cargo build --target wasm32-unknown-unknown
	- name: Build wasm32-wasi
	run: cargo build --target wasm32-wasi

	windows:
	name: cargo test LocalFileSystem (win64)
	runs-on: windows-latest
	defaults:
	run:
	working-directory: object_store
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: true
	- name: Run LocalFileSystem tests
	run: cargo test local::tests

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Fix multipart uploads with checksums on object locked buckets #4118

Workflow file

Fix multipart uploads with checksums on object locked buckets #4118

Jobs

Run details

Workflow file for this run