diff --git a/froster/froster.py b/froster/froster.py index 34a5e0e..66dd176 100755 --- a/froster/froster.py +++ b/froster/froster.py @@ -5294,7 +5294,7 @@ def _walker(self, top, skipdirs=['.snapshot',]): except Exception: print_error() - def _walkerr(oserr): + def _walkerr(self, oserr): """ error handler for os.walk """ print_error(str(oserr)) diff --git a/install.sh b/install.sh index 45eddd4..c8e7b37 100755 --- a/install.sh +++ b/install.sh @@ -7,11 +7,19 @@ set -e ### VARIABLES ### ################# -date_YYYYMMDDHHMMSS=$(date +%Y%m%d%H%M%S) # Get the current date in YYYYMMDD format - XDG_DATA_HOME=${XDG_DATA_HOME:-$HOME/.local/share} XDG_CONFIG_HOME=${XDG_CONFIG_HOME:-$HOME/.config} +date_YYYYMMDDHHMMSS=$(date +%Y%m%d%H%M%S) # Get the current date in YYYYMMDD format + +froster_data_dir=${XDG_DATA_HOME}/froster +froster_all_data_backups=${XDG_DATA_HOME}/froster_backups +froster_data_backup_dir=${froster_all_data_backups}/froster_${date_YYYYMMDDHHMMSS}.bak + +froster_config_dir=${XDG_CONFIG_HOME}/froster +froster_all_config_backups=${XDG_CONFIG_HOME}/froster_backups +froster_config_backup_dir=${froster_all_config_backups}/froster_${date_YYYYMMDDHHMMSS}.bak + ##################### ### ERROR HANDLER ### ##################### @@ -20,32 +28,22 @@ XDG_CONFIG_HOME=${XDG_CONFIG_HOME:-$HOME/.config} trap 'catch $? $BASH_COMMAND' EXIT catch() { - if [ "$1" != "0" ]; then - # error handling goes here - if [[ $(command -v pipx) ]]; then - if pipx list | grep 'froster' >/dev/null 2>&1; then - echo " Uninstalling froster..." - pipx uninstall froster >/dev/null 2>&1 - echo " ...uninstalled" - fi - fi + if [ "$1" != "0" ]; then + echo -e "\nError: $2: Installation failed!\n" # Restore (if any) backed up froster config files - if [[ -d ${XDG_CONFIG_HOME}/froster_${date_YYYYMMDDHHMMSS}.bak ]]; then - mv -f ${XDG_CONFIG_HOME}/froster_${date_YYYYMMDDHHMMSS}.bak ${XDG_CONFIG_HOME}/froster >/dev/null 2>&1 + if [[ -d ${froster_config_backup_dir} ]]; then + mv -f ${froster_config_backup_dir} ${froster_config_dir} >/dev/null 2>&1 fi # Restore (if any) backed up froster data files - if [[ -d ${XDG_DATA_HOME}/froster_${date_YYYYMMDDHHMMSS}.bak ]]; then - mv -f ${XDG_DATA_HOME}/froster_${date_YYYYMMDDHHMMSS}.bak ${XDG_DATA_HOME}/froster >/dev/null 2>&1 + if [[ -d ${froster_data_backup_dir} ]]; then + mv -f ${froster_data_backup_dir} ${froster_data_dir} >/dev/null 2>&1 fi rm -rf ${pwalk_path} >/dev/null 2>&1 rm -rf rclone-current-linux-*.zip rclone-v*/ >/dev/null 2>&1 - - echo - echo "Installation failed!" fi } @@ -71,7 +69,6 @@ spinner() { fi } - ################# ### FUNCTIONS ### ################# @@ -213,79 +210,46 @@ check_dependencies() { # Backup older installations (if any) but keep the froster-archive.json and config.ini files backup_old_installation() { - # Back up (if any) older froster data files - if [[ -d ${XDG_DATA_HOME}/froster ]]; then + # Make sure we did not left any backup files from previous updates. + # Move all backups to the data or config backup directories + mkdir -p ${froster_all_data_backups} + mkdir -p ${froster_all_config_backups} + find ${XDG_DATA_HOME} -maxdepth 1 -type d -name "froster_*.bak" -print0 | xargs -0 -I {} mv {} $froster_all_data_backups + find ${XDG_CONFIG_HOME} -maxdepth 1 -type d -name "froster_*.bak" -print0 | xargs -0 -I {} mv {} $froster_all_config_backups - backup=true - echo - echo "Backing up older froster installation..." + # Back up (if any) older froster data files + if [[ -d ${froster_data_dir} ]]; then + + echo -e "\nBacking Froster data folder ..." # Copy the froster directory to froster_YYYYMMDD.bak - cp -rf ${XDG_DATA_HOME}/froster ${XDG_DATA_HOME}/froster_${date_YYYYMMDDHHMMSS}.bak + cp -rf ${froster_data_dir} ${froster_data_backup_dir} - echo " Data back up at ${XDG_DATA_HOME}/froster_${date_YYYYMMDDHHMMSS}.bak" + echo " source: ${froster_data_dir}" + echo " destination: ${froster_data_backup_dir}" + + echo "...data backed up" fi # Back up (if any) older froster configurations - if [[ -d ${XDG_CONFIG_HOME}/froster ]]; then + if [[ -d ${froster_config_dir} ]]; then - if [ "$backup" != "true" ]; then - echo - echo "Backing up older froster installation..." - fi + echo -e "\nBacking Froster config folder ..." - backup=true + echo " source: ${froster_config_dir}" + echo " destination: ${froster_config_backup_dir}" # Move the froster config directory to froster.bak - cp -rf ${XDG_CONFIG_HOME}/froster ${XDG_CONFIG_HOME}/froster_${date_YYYYMMDDHHMMSS}.bak - - echo " Config back up at ${XDG_CONFIG_HOME}/froster_${date_YYYYMMDDHHMMSS}.bak" - fi - - if [ "$backup" = "true" ]; then - echo "...older froster installation backed up" - fi - - # Check if froster is already installed, if so uninstall it - if which froster >/dev/null 2>&1; then - echo - echo "Uninstalling existing froster installation..." - - if pipx list | grep froster >/dev/null 2>&1; then - pipx uninstall froster >/dev/null 2>&1 & - spinner $! - fi + cp -rf ${froster_config_dir} ${froster_config_backup_dir} - echo "...froster uninstalled" + echo "...config backed up" fi - - # Keep the froster-archives.json file (if any) - if [[ -f ${XDG_DATA_HOME}/froster_${date_YYYYMMDDHHMMSS}.bak/froster-archives.json ]]; then - echo - echo "Restoring Froster archives json data from backup..." - - # Create the froster directory if it does not exist - mkdir -p ${XDG_DATA_HOME}/froster - - # Copy the froster-archives.json file to the data directory - cp -f ${XDG_DATA_HOME}/froster_${date_YYYYMMDDHHMMSS}.bak/froster-archives.json ${XDG_DATA_HOME}/froster/froster-archives.json - - echo "...restored" - fi - - # Remove old files - rm -rf ${XDG_DATA_HOME}/froster - rm -rf ${XDG_CONFIG_HOME}/froster - rm -f ${HOME}/.local/bin/froster - rm -f ${HOME}/.local/bin/froster.py - rm -f ${HOME}/.local/bin/s3-restore.py } install_pipx() { - echo - echo "Installing pipx..." + echo -e "\nInstalling pipx..." # Check if pipx is installed if [[ -z $(command -v pipx) ]]; then @@ -306,67 +270,72 @@ install_pipx() { else echo "...pipx already installed" - echo - echo "Upgrading pipx..." + echo -e "\nUpgrading pipx..." pipx upgrade pipx >/dev/null 2>&1 & spinner $! echo "...pipx upgraded" - echo - echo "Ensuring path for pipx..." + echo -e "\nEnsuring path for pipx..." pipx ensurepath >/dev/null 2>&1 echo "...path ensured" fi # Check if PIPX_BIN_DIR is set and not empty, otherwise default to ~/.local/bin PIPX_BIN_DIR="${PIPX_BIN_DIR:-$HOME/.local/bin}" - echo - echo "Adding $PIPX_BIN_DIR to PATH for this installation session" + echo -e "\nAdding $PIPX_BIN_DIR to PATH for this installation session" export PATH="$PATH:$PIPX_BIN_DIR" } install_froster() { - echo - echo "Installing latest version of froster..." + echo -e "\nRemoving old froster files..." + rm -rf ${froster_data_dir} + rm -rf ${froster_config_dir} + rm -f ${HOME}/.local/bin/froster + rm -f ${HOME}/.local/bin/froster.py + rm -f ${HOME}/.local/bin/s3-restore.py + echo "...old froster files removed" if [ "$LOCAL_INSTALL" = "true" ]; then - echo " Installing from the current directory in --editable mode" + echo -e "\nInstalling Froster from the current directory in --editable mode..." pip install -e . >/dev/null 2>&1 & spinner $! + echo "...Froster installed" else - echo " Installing from PyPi package repository" + if pipx list | grep froster >/dev/null 2>&1; then + echo -e "\nUninstalling old Froster..." + pipx uninstall froster >/dev/null 2>&1 + echo "...old Froster uninstalled" + fi + + echo -e "\nInstalling Froster from PyPi package repository" pipx install froster >/dev/null 2>&1 & spinner $! + echo "...Froster installed" fi - echo " Installation path: $(which froster)" - # Keep the config.ini file (if any) - if [[ -f ${XDG_CONFIG_HOME}/froster_${date_YYYYMMDDHHMMSS}.bak/config.ini ]]; then + if [[ -f ${froster_config_backup_dir}/config.ini ]]; then # Create the froster directory if it does not exist - mkdir -p ${XDG_CONFIG_HOME}/froster + mkdir -p ${froster_config_dir} # Copy the config file to the data directory - cp -f ${XDG_CONFIG_HOME}/froster_${date_YYYYMMDDHHMMSS}.bak/config.ini ${XDG_CONFIG_HOME}/froster/config.ini + cp -f ${froster_config_backup_dir}/config.ini ${froster_config_dir} fi # Keep the froster-archives.json file (if any) - if [[ -f ${XDG_CONFIG_HOME}/froster_${date_YYYYMMDDHHMMSS}.bak/froster-archives.json ]]; then + if [[ -f ${froster_data_backup_dir}/froster-archives.json ]]; then # Create the froster directory if it does not exist - mkdir -p ${XDG_DATA_HOME}/froster + mkdir -p ${froster_data_dir} # Copy the froster-archives.json file to the data directory - cp -f ${XDG_CONFIG_HOME}/froster_${date_YYYYMMDDHHMMSS}.bak/froster-archives.json ${XDG_DATA_HOME}/froster/froster-archives.json + cp -f ${froster_data_backup_dir}/froster-archives.json ${froster_data_dir} fi - - echo "...froster installed" } install_pwalk() { - echo - echo "Installing third-party dependency: pwalk... " + echo -e "\nInstalling third-party dependency: pwalk... " # Variables of pwalk third-party tool froster is using pwalk_commit=1df438e9345487b9c51d1eea3c93611e9198f173 # update this commit when new pwalk version released @@ -404,8 +373,7 @@ install_pwalk() { # Install rclone install_rclone() { - echo - echo "Installing third-party dependency: rclone... " + echo -e "\nInstalling third-party dependency: rclone... " # Check the architecture of the system arch=$(uname -m) @@ -479,12 +447,11 @@ install_rclone version=$(froster -v | awk '{print $2}') # Print success message -echo -echo "froster $version has been successfully installed!" -echo +echo -e "\n\nSUCCESS!" + +echo -e "\nFroster version: $version" +echo -e "Installation path: $(which froster)" # Print post-installation instructions -echo -echo "You will need to open a new terminal or refresh your current terminal session by running command:" -echo " source ~/.bashrc" -echo \ No newline at end of file +echo -e "\n\nYou will need to open a new terminal or refresh your current terminal session by running command:" +echo -e " source ~/.bashrc\n" \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 9369e88..22734e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "froster" -version = "0.17.2" +version = "0.18.0" description = "Froster is a tool for easy data transfer between local file systems and AWS S3 storage." authors = ["Victor Machado "] readme = "README.md" diff --git a/s3-restore.sh b/s3-restore.sh index 7f0498f..3809d5c 100755 --- a/s3-restore.sh +++ b/s3-restore.sh @@ -20,11 +20,11 @@ # Check your AWS PROFILE at ~/.aws/credentials and ~/.aws/config # The output of rclone command should be shown in a few minuts +export RCLONE_S3_PROFILE= +export RCLONE_S3_REGION= export RCLONE_S3_PROVIDER= export RCLONE_S3_ENDPOINT= -export RCLONE_S3_REGION= export RCLONE_S3_LOCATION_CONSTRAINT= -export RCLONE_S3_PROFILE= export AWS_ACCESS_KEY_ID= export AWS_SECRET_ACCESS_KEY= @@ -86,21 +86,22 @@ check_environment() { if [[ -z $RCLONE_S3_REGION ]]; then echo -e "\nWARNING: RCLONE_S3_REGION not set. Set it in the install.sh top's variables.\n" + echo -e "If your S3 provider does not need a region, you can ignore this warning.\n" echo -e "You may get this information from the Where-did-the-files-go.txt manifest file\n" fi if [[ -z $RCLONE_S3_LOCATION_CONSTRAINT ]]; then echo -e "\n WARNING: RCLONE_S3_LOCATION_CONSTRAINT not set. Set it in the install.sh top's variables.\n" + echo -e "If your S3 provider does not need a region, you can ignore this warning.\n" echo -e "You may get this information from the Where-did-the-files-go.txt manifest file\n" fi if [[ -z $RCLONE_S3_PROFILE ]]; then - if [[ -z $AWS_ACCESS_KEY_ID || -z $AWS_SECRET_ACCESS_KEY ]]; then - echo -e "\nAWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY not set. Set it in the install.sh top's variables.\n" - echo -e "You may get this information from the Where-did-the-files-go.txt manifest file\n" - exit 1 - else - echo -e "\nRCLONE_S3_PROFILE not set. Set it in the install.sh top's variables.\n" + if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ]; then + echo -e "\nRclone needs to configure one of two options:" + echo " 1) RCLONE_S3_PROFILE" + echo " 2) AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY\n" + echo -e "\nSet one of them it in the install.sh top's variables.\n" echo -e "You may get this information from the Where-did-the-files-go.txt manifest file\n" exit 1 fi @@ -154,7 +155,7 @@ restore_rclone(){ echo -e "\t[-]${RCLONE_S3_PROFILE} profile not found in ~/.aws/credentials. Add credentials to restore.conf.\n" exit 1 else - echo -e "\t\t[+] Using configured AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY as AWS credentials\n" + echo -e "\t\t[+] Using configured AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY as AWS credentials" fi else echo -e "\t\t[+] Using ~/.aws/credentials profile:${RCLONE_S3_PROFILE}" @@ -204,10 +205,16 @@ restore_rclone(){ # Execute the restore command rclone backend restore --max-depth=1 -o priority=Bulk -o lifetime=30 ${ARCHIVE_FOLDER} > ${RESTORE_OUTPUT} - + # Get the status of each file statuses=$(jq -r '.[].Status' ${RESTORE_OUTPUT}) + # Check if the retrieve command failed + if [ -z "$statuses" ]; then + echo -e "\nError: Retrieve command failed. Check the configured variables at the top of the script.\n" + exit 1 + fi + # Variable to store the status of the restore all_ok=true @@ -219,12 +226,12 @@ restore_rclone(){ fi done + if $all_ok; then echo -e "\nGlacier retrieve initiated." echo -e "Execute the same command again 48 hours after restore was initiated.\n" exit 0 else - # Retrieval already initiated # Execute the restore-status command rclone backend restore-status --max-depth=1 -o priority=Bulk -o lifetime=30 ${ARCHIVE_FOLDER} > ${RESTORE_STATUS_OUTPUT} @@ -251,27 +258,40 @@ restore_rclone(){ done if $all_true; then - echo -e "\nRetrieve: Glacier retrieve in progress, try again 48 hours after restore was initiated.\n" + echo -e "\nINFO: Glacier retrieve in progress, try again 48 hours after restore was initiated.\n" exit 0 elif $all_false; then # If all files have been restored, we can proceed with the restore # This path is the only one that goes though the restore process - echo -e "\nRetrieve: Glacier retrieve finished." + echo -e "\nINFO: Glacier retrieve finished." else - echo -e "\nRetrieve: Only some files have been retrieved. Glacier retrieve in progress, try again 48 hours after restore was initiated.\n" + echo -e "\nINFO: Only some files have been retrieved. Glacier retrieve in progress, try again 48 hours after restore was initiated.\n" exit 0 fi fi fi ### Restoring from S3 to local folder - echo "Restoring from ${ARCHIVE_FOLDER} to ${DIRECTORY_PATH} ..." + echo -e "\nRestoring from ${ARCHIVE_FOLDER} to ${DIRECTORY_PATH} ..." rclone copy --checksum --progress --verbose ${ARCHIVE_FOLDER} ${DIRECTORY_PATH} ${DEPTH} ### Comparing S3 with local folder - echo "Running Checksum comparison, hit ctrl+c to cancel ... " - rclone check --verbose --exclude='.froster.md5sum' ${ARCHIVE_FOLDER} ${DIRECTORY_PATH} ${DEPTH} + echo -e "\nRunning Checksum comparison, hit ctrl+c to cancel ... " + rclone check --verbose --exclude='.froster.md5sum' --exclude='Where-did-the-files-go.txt' ${ARCHIVE_FOLDER} ${DIRECTORY_PATH} ${DEPTH} + rclone_checksum_result=$? + + if [ $rclone_checksum_result -eq 0 ]; then + echo -e "\nChecksum verification: success" + echo -e "Download successfull!" + elif [ $rclone_checksum_result -eq 1 ]; then + echo "Error: Checksums do not match." + exit 1 + else + echo "Error: rclone error code: $rclone_checksum_result." + exit 1 + fi + ### After restore we must check if there are any files to untar ## Find all tar files and store them in an array mapfile -t tar_files < <(find "${DIRECTORY_PATH}" -type f -name "${TAR_FILENAME}") @@ -290,11 +310,14 @@ restore_rclone(){ echo "Deleted: $tar_file" else echo "Failed to extract: $tar_file" + exit 1 fi # Change back to the original directory cd - >/dev/null || return done cd "$cdir" + + echo -e "\nRESTORE SUCCESSFULLY COMPLETED!" } ########