Skip to content

Commit

Permalink
add cleanup for outdated reports (#350)
Browse files Browse the repository at this point in the history
  • Loading branch information
@vbragin
vbragin authored Dec 11, 2024
1 parent 67ba947 commit 0a4e633
Show file tree
Hide file tree
Showing 16 changed files with 183 additions and 166 deletions.
58 changes: 48 additions & 10 deletions src/main/java/io/qameta/allure/bamboo/AllureArtifactsManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
import com.atlassian.sal.api.ApplicationProperties;
import com.atlassian.sal.api.UrlMode;
import com.google.common.collect.ImmutableList;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.tools.ant.types.FileSet;
import org.jetbrains.annotations.NotNull;
Expand All @@ -61,11 +62,14 @@
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
Expand All @@ -88,14 +92,15 @@
import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.toMap;
import static javax.ws.rs.core.UriBuilder.fromPath;
import static org.apache.commons.io.FileUtils.deleteQuietly;
import static org.apache.commons.io.FileUtils.forceMkdir;
import static org.apache.commons.io.FileUtils.moveDirectory;
import static org.apache.commons.lang3.StringUtils.isEmpty;
import static org.codehaus.plexus.util.FileUtils.copyDirectory;
import static org.codehaus.plexus.util.FileUtils.copyURLToFile;

@SuppressWarnings({"ClassDataAbstractionCoupling", "PMD.AvoidInstantiatingObjectsInLoops", "PMD.GodClass"})
@SuppressWarnings({
"ClassDataAbstractionCoupling",
"PMD.AvoidInstantiatingObjectsInLoops",
"PMD.GodClass"
})
public class AllureArtifactsManager {

private static final Logger LOGGER = LoggerFactory.getLogger(AllureArtifactsManager.class);
Expand Down Expand Up @@ -181,7 +186,7 @@ private String getLocalStorageURL(final String planKeyString,
final String buildNumber,
final String filePath) {
try {
final File file = getLocalStoragePath(planKeyString, buildNumber).resolve(filePath).toFile();
final File file = getLocalStorageReportPath(planKeyString, buildNumber).resolve(filePath).toFile();
final String fullPath = (file.isDirectory())
? new File(file, INDEX_HTML).getAbsolutePath() : file.getAbsolutePath();
return new File(fullPath).toURI().toURL().toString();
Expand Down Expand Up @@ -215,7 +220,7 @@ Collection<Path> downloadAllArtifactsTo(final @NotNull ChainResultsSummary chain
artifact.getLabel(), artifactName,
chainResultsSummary.getPlanKey(), chainResultsSummary.getBuildNumber());
final File stageDir = new File(baseDir, UUID.randomUUID().toString());
forceMkdir(stageDir);
FileUtils.forceMkdir(stageDir);
resultsPaths.add(stageDir.toPath());
final ArtifactLinkDataProvider dataProvider
= artifactLinkManager.getArtifactLinkDataProvider(artifact);
Expand Down Expand Up @@ -304,11 +309,11 @@ Optional<AllureBuildResult> uploadReportArtifacts(final @NotNull ImmutableChain
if (isAgentArtifactHandler(artifactHandler)) {
final String planKey = chain.getPlanKey().getKey();
final String buildNumber = String.valueOf(summary.getBuildNumber());
final File destDir = getLocalStoragePath(planKey, buildNumber).toFile();
final File destDir = getLocalStorageReportPath(planKey, buildNumber).toFile();
if (destDir.exists()) {
deleteQuietly(destDir);
FileUtils.deleteQuietly(destDir);
}
moveDirectory(reportDir, destDir);
FileUtils.moveDirectory(reportDir, destDir);
return Optional.of(allureBuildResult(true, null)
.withHandlerClass(artifactHandler.getClass().getName()));
}
Expand Down Expand Up @@ -344,10 +349,43 @@ public ArtifactHandlerPublishingResult call() {
return Optional.empty();
}

private Path getLocalStoragePath(final String planKey, final String buildNumber) {
void cleanupOldReportArtifacts(final @NotNull ImmutableChain chain,
final Integer maxStoredReportsCount) {
if (maxStoredReportsCount == null || maxStoredReportsCount <= 0) {
return;
}
for (final ArtifactHandler artifactHandler : getArtifactHandlers()) {
if (!isAgentArtifactHandler(artifactHandler)) {
continue;
}
final String planKey = chain.getPlanKey().getKey();
final Path reports = getLocalStoragePlanReportsPath(planKey);
final List<Long> buildNumbers = new ArrayList<>();
try (DirectoryStream<Path> ds = Files.newDirectoryStream(reports, Files::isDirectory)) {
for (Path p : ds) {
if (!StringUtils.isNumeric(p.getFileName().toString())) {
continue;
}
buildNumbers.add(Long.parseLong(p.getFileName().toString()));
}
} catch (Exception e) {
LOGGER.error("Failed to clean up old Allure Report", e);
}
buildNumbers.stream()
.sorted(Comparator.reverseOrder())
.skip(maxStoredReportsCount)
.forEach(bn -> FileUtils.deleteQuietly(reports.resolve(bn.toString()).toFile()));
}
}

private Path getLocalStorageReportPath(final String planKey, final String buildNumber) {
return Paths.get(settingsManager.getSettings().getLocalStoragePath(), REPORTS_SUBDIR, planKey, buildNumber);
}

private Path getLocalStoragePlanReportsPath(final String planKey) {
return Paths.get(settingsManager.getSettings().getLocalStoragePath(), REPORTS_SUBDIR, planKey);
}

private void logAndThrow(final Exception e,
final String message) {
LOGGER.error(message, e);
Expand Down
5 changes: 5 additions & 0 deletions src/main/java/io/qameta/allure/bamboo/AllureBuildCompleteAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,11 @@ private void buildReport(final Path artifactsTempDir,
.ifPresent(result -> result.dumpToCustomData(customBuildData));
}
FileUtils.deleteQuietly(copyPath.toFile());
if (globalConfig.isEnabledReportsCleanup()
&& buildConfig.getMaxStoredReportsCount() != null
&& buildConfig.getMaxStoredReportsCount() > 0) {
artifactsManager.cleanupOldReportArtifacts(chain, buildConfig.getMaxStoredReportsCount());
}
} catch (Exception e) {
LOGGER.error("Failed to build allure report for {}", chain.getName(), e);
allureBuildResult(false, stackTraceToString(e)).dumpToCustomData(customBuildData);
Expand Down
36 changes: 28 additions & 8 deletions src/main/java/io/qameta/allure/bamboo/AllureBuildConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,39 +16,53 @@
package io.qameta.allure.bamboo;

import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;

import javax.annotation.Nullable;
import java.io.Serializable;
import java.util.Map;

import static io.qameta.allure.bamboo.AllureConstants.ALLURE_CONFIG_ARTIFACT_NAME;
import static io.qameta.allure.bamboo.AllureConstants.ALLURE_CONFIG_CUSTOM_LOGO_PATH;
import static io.qameta.allure.bamboo.AllureConstants.ALLURE_CONFIG_ENABLED;
import static io.qameta.allure.bamboo.AllureConstants.ALLURE_CONFIG_EXECUTABLE;
import static io.qameta.allure.bamboo.AllureConstants.ALLURE_CONFIG_FAILED_ONLY;
import static io.qameta.allure.bamboo.AllureConstants.ALLURE_CUSTOM_LOGO_PATH;
import static io.qameta.allure.bamboo.AllureConstants.ALLURE_CONFIG_MAX_STORED_REPORTS_COUNT;
import static java.lang.Boolean.FALSE;
import static java.lang.Boolean.TRUE;
import static java.util.Optional.ofNullable;

public final class AllureBuildConfig implements Serializable {

private static final long serialVersionUID = 1L;

private static final String DEFAULT_ARTIFACT_NAME = "allure-results";
private static final String DEFAULT_CUSTOM_LOGO_URL = "https://qameta.io/allure-report/img/reportlogo.svg";

private final boolean onlyForFailed;
private final String executable;
private final boolean enabled;
private final String artifactName;
private final String logoUrl;
private static final String DEFAULT_ARTIFACT_NAME = "allure-results";
public static final String DEFAULT_CUSTOM_LOGO_URL = "https://qameta.io/allure-report/img/reportlogo.svg";
private final Integer maxStoredReportsCount;

private AllureBuildConfig(final String executable,
final String enabled,
final String onlyForFailed,
final String artifactName,
final String logoUrl) {
this.onlyForFailed = StringUtils.isEmpty(onlyForFailed) ? TRUE : Boolean.parseBoolean(onlyForFailed);
this.enabled = StringUtils.isEmpty(enabled) ? FALSE : Boolean.parseBoolean(enabled);
final String logoUrl,
final String maxStoredReportsCount) {
this.onlyForFailed = StringUtils.isBlank(onlyForFailed)
? TRUE : Boolean.parseBoolean(onlyForFailed);
this.enabled = StringUtils.isBlank(enabled)
? FALSE : Boolean.parseBoolean(enabled);
this.executable = executable;
this.artifactName = artifactName;
this.logoUrl = !logoUrl.isEmpty() ? logoUrl : AllureBuildConfig.DEFAULT_CUSTOM_LOGO_URL;
this.logoUrl = StringUtils.isBlank(logoUrl)
? AllureBuildConfig.DEFAULT_CUSTOM_LOGO_URL : logoUrl;
this.maxStoredReportsCount =
StringUtils.isBlank(maxStoredReportsCount) || !StringUtils.isNumeric(maxStoredReportsCount)
? -1 : NumberUtils.toInt(maxStoredReportsCount);
}

static AllureBuildConfig fromContext(final Map<String, String> context) {
Expand All @@ -57,7 +71,9 @@ static AllureBuildConfig fromContext(final Map<String, String> context) {
getSingleValue(context, ALLURE_CONFIG_ENABLED, FALSE.toString()),
getSingleValue(context, ALLURE_CONFIG_FAILED_ONLY, FALSE.toString()),
getSingleValue(context, ALLURE_CONFIG_ARTIFACT_NAME, AllureBuildConfig.DEFAULT_ARTIFACT_NAME),
getSingleValue(context, ALLURE_CUSTOM_LOGO_PATH, AllureBuildConfig.DEFAULT_CUSTOM_LOGO_URL));
getSingleValue(context, ALLURE_CONFIG_CUSTOM_LOGO_PATH, AllureBuildConfig.DEFAULT_CUSTOM_LOGO_URL),
getSingleValue(context, ALLURE_CONFIG_MAX_STORED_REPORTS_COUNT, null)
);
}

@Nullable
Expand Down Expand Up @@ -93,4 +109,8 @@ public String getCustomLogoUrl() {
return this.logoUrl;
}

public Integer getMaxStoredReportsCount() {
return maxStoredReportsCount;
}

}
3 changes: 3 additions & 0 deletions src/main/java/io/qameta/allure/bamboo/AllureBuildResult.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@
import static org.apache.commons.lang3.StringUtils.isEmpty;

class AllureBuildResult implements Serializable {

private static final long serialVersionUID = 1L;

private final boolean success;
private String artifactHandlerClass;
private String failureDetails;
Expand Down
4 changes: 3 additions & 1 deletion src/main/java/io/qameta/allure/bamboo/AllureCommandLineSupport.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,9 @@
import static org.apache.commons.lang3.SystemUtils.IS_OS_WINDOWS;

public class AllureCommandLineSupport {
private static final Pattern RESULT_TC_COUNT_REGEX = Pattern.compile(".+Found (\\d+) test cases.+", Pattern.DOTALL);

private static final Pattern RESULT_TC_COUNT_REGEX
= Pattern.compile(".+Found (\\d+) test cases.+", Pattern.DOTALL);
private static final int GENERATE_TIMEOUT_MS = (int) MINUTES.toMillis(10);

String runCommand(final String cmd, final String... args) {
Expand Down
22 changes: 7 additions & 15 deletions src/main/java/io/qameta/allure/bamboo/AllureConstants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,22 +20,10 @@
*/
final class AllureConstants {

/**
* The name of artifact.
*/
static final String ALLURE_ARTIFACT_NAME = "Allure Report";
/**
* The executable prefix.
*/
static final String ALLURE_EXECUTION_PREFIX = "system.builder.allure";
/**
* The directory with allure results relative from build directory.
*/
static final String ALLURE_CONFIG_RESULTS_PATH = "custom.allure.config.results.path";
/**
* The subdirectory (or subdirectories) to put generated report into.
*/
static final String ALLURE_CONFIG_REPORT_PATH = "custom.allure.config.report.path";
/**
* The name of executable.
*/
Expand All @@ -47,17 +35,21 @@ final class AllureConstants {
static final String ALLURE_CONFIG_ENABLED = "custom.allure.config.enabled";
static final String ALLURE_CONFIG_FAILED_ONLY = "custom.allure.config.failed.only";
static final String ALLURE_CONFIG_ARTIFACT_NAME = "custom.allure.artifact.name";
static final String ALLURE_CONFIG_STORAGE_TYPE = "custom.allure.config.storage.type";
static final String ALLURE_CONFIG_DOWNLOAD_ENABLED = "custom.allure.config.download.enabled";
static final String ALLURE_CONFIG_ENABLED_BY_DEFAULT = "custom.allure.config.enabled.default";
static final String ALLURE_CONFIG_DOWNLOAD_URL = "custom.allure.config.download.url";
static final String ALLURE_CONFIG_DOWNLOAD_CLI_URL = "custom.allure.config.download.cli.url";
static final String ALLURE_CONFIG_LOCAL_STORAGE = "custom.allure.config.local.storage";

// ALLURE CUSTOM LOGO
static final String ALLURE_CUSTOM_LOGO_ENABLED = "custom.allure.config.logo.enabled";
static final String ALLURE_CUSTOM_LOGO_PATH = "custom.allure.logo.url";
static final String ALLURE_CONFIG_CUSTOM_LOGO_ENABLED = "custom.allure.config.logo.enabled";
static final String ALLURE_CONFIG_CUSTOM_LOGO_PATH = "custom.allure.logo.url";

// ALLURE CLEANUP OLD REPORTS
static final String ALLURE_CONFIG_REPORTS_CLEANUP_ENABLED = "custom.allure.config.reports.cleanup.enabled";
static final String ALLURE_CONFIG_MAX_STORED_REPORTS_COUNT = "custom.allure.max.stored.reports.count";

private AllureConstants() {
// do not instantiate
}
}
1 change: 1 addition & 0 deletions src/main/java/io/qameta/allure/bamboo/AllureDownloader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import static org.apache.commons.io.FileUtils.moveDirectory;

class AllureDownloader {

private static final Logger LOGGER = LoggerFactory.getLogger(AllureDownloader.class);

private final AllureSettingsManager settingsManager;
Expand Down
Loading

0 comments on commit 0a4e633

Please sign in to comment.