From af3dd27c90277aaf792feb1f57f44f82d6183b7d Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Mon, 18 Sep 2023 10:04:04 -0700 Subject: [PATCH] Squashed commit of the following: commit 016c59fd4c6e4b2eba25252d7739380564932966 Author: Dima Ryazanov Date: Mon Sep 18 07:16:20 2023 -0700 Bump quiltcore and use workflows commit 843d22b7114dbba7dbd36c7aa29547d592111028 Author: Ernest Prabhakar Date: Mon Sep 11 15:35:28 2023 -0700 re-enabled failing workflow tests commit 41d6063dce7d1b3c41362eac4011f5f0e4007e37 Author: Dr. Ernie Prabhakar Date: Mon Sep 11 15:23:07 2023 -0700 Quiltcore java preview (#122) Ernie's updates for Sep 6-11 - Added unit and integration tests - Rewrote package to NOT pre-install (instead use Manifest to find relativeChildren) --------- Co-authored-by: Dima Ryazanov Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/pkg-test.yml | 61 +++++++++++++++++++ plugins/nf-quilt/build.gradle | 2 +- .../main/nextflow/quilt/QuiltProduct.groovy | 8 ++- .../nextflow/quilt/jep/QuiltPackage.groovy | 2 +- .../quilt/nio/QuiltFileSystemProvider.groovy | 39 ++++++++++-- .../quilt/jep/QuiltPackageTest.groovy | 35 ++++++++--- .../nio/QuiltFileSystemProviderTest.groovy | 12 ++++ .../nextflow/quilt/nio/QuiltNioTest.groovy | 16 ++--- 8 files changed, 153 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/pkg-test.yml diff --git a/.github/workflows/pkg-test.yml b/.github/workflows/pkg-test.yml new file mode 100644 index 00000000..51317a7c --- /dev/null +++ b/.github/workflows/pkg-test.yml @@ -0,0 +1,61 @@ +name: Test + +on: + # Trigger at every push. Action will also be visible from Pull Requests to master + push: # Comment this line to trigger action only on pull-requests (not recommended if you don't pay for GH Actions) + pull_request: + branches: [master] + +permissions: read-all + +jobs: + build: + name: Package Test + permissions: + contents: read + id-token: write + issues: write + pull-requests: write + + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + java_version: [19] + runs-on: ${{ matrix.os }} + + steps: + # Git Checkout + - name: Checkout Code + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + fetch-depth: 0 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v3 + with: + role-to-assume: arn:aws:iam::712023778557:role/GitHub-Testing-NF-Quilt + aws-region: us-east-1 + + - name: Setup Java ${{matrix.java_version}} + uses: actions/setup-java@v3 + with: + java-version: ${{matrix.java_version}} + distribution: 'temurin' + architecture: x64 + cache: gradle + + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 + + - name: Run Package Tests + run: make pkg-test + + - name: Archive production artifacts + if: ${{ success() }} || ${{ failure() }} + uses: actions/upload-artifact@v3 + with: + name: nf-quilt-pkg-test + path: | + /home/runner/work/nf-quilt/nf-quilt/plugins/nf-quilt/build/reports/tests/test/ diff --git a/plugins/nf-quilt/build.gradle b/plugins/nf-quilt/build.gradle index 87a98eb1..8186de68 100644 --- a/plugins/nf-quilt/build.gradle +++ b/plugins/nf-quilt/build.gradle @@ -61,7 +61,7 @@ ext{ dependencies { // quiltcore - implementation 'com.quiltdata.quiltcore:quiltcore:0.0.2' + implementation 'com.quiltdata.quiltcore:quiltcore:0.0.3' // This dependency is exported to consumers, that is to say found on their compile classpath. compileOnly "io.nextflow:nextflow:$nextflowVersion" diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy index 6a14192b..25318104 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy @@ -80,11 +80,17 @@ ${meta['workflow']['stats']['processes']} static void writeString(String text, QuiltPackage pkg, String filename) { String dir = pkg.packageDest() Path path = Paths.get(dir, filename.split('/') as String[]) + File parent = path.getParent().toFile() + if (parent != null && !parent.exists() && !parent.mkdirs()) { + throw new IllegalStateException("Couldn't create dir: " + parent); + } + try { Files.write(path, text.bytes) } catch (Exception e) { - log.error("writeString: cannot write `$text` to `$path` for `${pkg}`") + log.error("writeString[${e.getMessage()}]: fail write `$path` for `${pkg}`") + e.printStackTrace() } } diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy index 54bd7874..5d58e58f 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy @@ -254,7 +254,7 @@ class QuiltPackage { Manifest m = builder.build() try { - m.push(namespace, "nf-quilt:${today()}-${msg}") + m.push(namespace, "nf-quilt:${today()}-${msg}", parsed.workflowName) } catch (IOException e) { return 1 } diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy index b7ae4c53..972b072d 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy @@ -43,6 +43,7 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.Global import nextflow.Session +import nextflow.file.FileSystemTransferAware import nextflow.quilt.jep.QuiltParser import nextflow.quilt.jep.QuiltPackage @@ -54,7 +55,7 @@ import nextflow.quilt.jep.QuiltPackage @Slf4j @CompileStatic -class QuiltFileSystemProvider extends FileSystemProvider { +class QuiltFileSystemProvider extends FileSystemProvider implements FileSystemTransferAware { private final Map myEnv = new HashMap<>(System.getenv()) private final Map fileSystems = [:] @@ -84,6 +85,36 @@ class QuiltFileSystemProvider extends FileSystemProvider { return path.getFileSystem().provider() } + static boolean isLocalProvider(Path path) { + FileSystemProvider provider = provider(path) + String providerName = provider?.class?.name?.toLowerCase() ?: 'N/A' + println("QuiltFileSystemProvider.isLocalProvider[${path}] -> ${providerName}") + return providerName.contains("xfile") || providerName.contains("win") ||\ + providerName.contains("fat") || providerName == 'N/A' + } + + boolean canDownload(Path source, Path target) { + log.debug("QuiltFileSystemProvider.canDownload[${source}] -> ${target}") + return isLocalProvider(target) && source instanceof QuiltPath + } + + boolean canUpload(Path source, Path target) { + log.debug("QuiltFileSystemProvider.canUpload[${source}] -> ${target}") + return isLocalProvider(source) && target instanceof QuiltPath + } + + void download(Path source, Path target, CopyOption... options) throws IOException { + QuiltPath qSource = asQuiltPath(source) + Path local_source = qSource.localPath() + Files.copy(local_source, target, options) + } + + void upload(Path source, Path target, CopyOption... options) throws IOException { + QuiltPath qTarget = asQuiltPath(target) + Path local_target = qTarget.localPath() + Files.copy(source, local_target, options) + } + /** * @inheritDoc */ @@ -340,7 +371,7 @@ class QuiltFileSystemProvider extends FileSystemProvider { @Override void copy(Path from, Path to, CopyOption... options) throws IOException { - //log.debug("Attempting `copy`: ${from} -> ${to}") + log.debug("Attempting `copy`: ${from} -> ${to}") assert provider(from) == provider(to) if (from == to) { return // nothing to do -- just return @@ -366,7 +397,7 @@ class QuiltFileSystemProvider extends FileSystemProvider { @Override boolean isHidden(Path path) throws IOException { - return path.getFileName()?.toString()?.startsWith('.') + return path.getFileName()?.toString()?.contains('.') } @Override @@ -400,7 +431,7 @@ class QuiltFileSystemProvider extends FileSystemProvider { @Override def A readAttributes(Path path, Class type, LinkOption... options) throws IOException { - //log.debug 'BasicFileAttributes QuiltFileSystemProvider.readAttributes()' + log.debug 'BasicFileAttributes QuiltFileSystemProvider.readAttributes()' def attr = attributesCache.get(path) if (attr) { return attr diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy index f56472e9..1eab263c 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy @@ -106,6 +106,30 @@ class QuiltPackageTest extends QuiltSpecification { Files.exists(installPath) } + void 'should copy temp files into install folder'() { + given: + String filename = 'test.txt' + Path installPath = pkg.packageDest() + Path tempFile = File.createTempFile('test', '.txt').toPath() + Path installedFile = Paths.get(installPath.toString(), filename) + expect: + Files.exists(tempFile) + Files.exists(installPath) + !Files.exists(installedFile) + Files.copy(tempFile, installedFile) + Files.exists(installedFile) + } + + void 'should copy package files to temp Path'() { + given: + Path installPath = pkg.packageDest() + expect: + Files.exists(installPath) + Files.isDirectory(installPath) + Files.readAttributes(installPath, BasicFileAttributes) + } + + void 'should get attributes for package folder'() { given: def root = qpath.getRoot() @@ -126,6 +150,7 @@ class QuiltPackageTest extends QuiltSpecification { Files.readAttributes(qpath, BasicFileAttributes) } + void 'should return null on failed install'() { given: def url2 = TEST_URL.replace('quilt-', 'quilted-') @@ -140,17 +165,13 @@ class QuiltPackageTest extends QuiltSpecification { void 'should deinstall files'() { expect: Files.exists(qpath.localPath(true)) + Files.readAttributes(qpath, BasicFileAttributes) when: qpath.deinstall() then: !Files.exists(qpath.localPath(false)) - /* when: - Files.readAttributes(qpath, BasicFileAttributes) - then: - thrown(java.nio.file.NoSuchFileException) */ } - @Ignore() void 'should iterate over installed files '() { given: def root = qpath.getRoot() @@ -212,7 +233,7 @@ class QuiltPackageTest extends QuiltSpecification { opkg.push('msg', meta) == 0 } - @Ignore('QuiltCore-java does not support workflows yet') + // @Ignore('QuiltCore-java does not support workflows yet') @IgnoreIf({ env.WRITE_BUCKET == 'quilt-example' || env.WRITE_BUCKET == null }) void 'should fail if invalid workflow'() { given: @@ -222,7 +243,7 @@ class QuiltPackageTest extends QuiltSpecification { bad_wf.push('missing-workflow first time', [:]) == 1 } - @Ignore('QuiltCore-java does not support workflows yet') + // @Ignore('QuiltCore-java does not support workflows yet') @IgnoreIf({ env.WRITE_BUCKET == 'quilt-example' || env.WRITE_BUCKET == null }) void 'should fail push if unsatisfied workflow'() { given: diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy index fc18641e..0f841007 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy @@ -3,6 +3,8 @@ package nextflow.quilt.nio import nextflow.quilt.QuiltSpecification import groovy.transform.CompileDynamic +import java.nio.file.Path +import java.nio.file.Paths /** * @@ -22,4 +24,14 @@ class QuiltFileSystemProviderTest extends QuiltSpecification { // newDirectoryStream returns package path for write // do we need a new schema for quilt+local? + void 'should recognize isLocalProvider'() { + given: + Path local = File.createTempFile('test', '.txt').toPath() + Path remote = Paths.get(new URI(fullURL)) + + expect: + QuiltFileSystemProvider.isLocalProvider(local) == true + QuiltFileSystemProvider.isLocalProvider(remote) == false + } + } diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy index 9720176d..02450348 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy @@ -79,7 +79,7 @@ class QuiltNioTest extends QuiltSpecification { text.startsWith('id') } - @IgnoreIf({ System.getProperty('os.name').contains('ux') }) + // @IgnoreIf({ System.getProperty('os.name').contains('ux') }) @IgnoreIf({ System.getProperty('os.name').contains('indows') }) void 'should read file attributes'() { given: @@ -132,7 +132,7 @@ class QuiltNioTest extends QuiltSpecification { then: !attrs.isRegularFile() attrs.isDirectory() - attrs.size() == 128 + attrs.size() > 100 // differs by platform !attrs.isSymbolicLink() !attrs.isOther() attrs.fileKey() == root @@ -148,7 +148,7 @@ class QuiltNioTest extends QuiltSpecification { then: !attrs.isRegularFile() attrs.isDirectory() - attrs.size() == 224 + attrs.size() > 100 // differs by platform !attrs.isSymbolicLink() !attrs.isOther() attrs.fileKey() == '/' @@ -197,7 +197,7 @@ class QuiltNioTest extends QuiltSpecification { if (source) { Files.delete(source) } } - @Ignore + @IgnoreIf({ env.WRITE_BUCKET == 'quilt-example' || env.WRITE_BUCKET == null }) void 'copy a remote file to a bucket'() { given: Path path = Paths.get(new URI(WRITE_URL)) @@ -212,11 +212,11 @@ class QuiltNioTest extends QuiltSpecification { readObject(path).trim() == TEXT } - @Ignore + @Ignore('QuiltFileSystem.copy not implemented') void 'move a remote file to a bucket'() { given: Path path = Paths.get(new URI(WRITE_URL)) - final source_url = WRITE_URL.replace('test_folder', 'source') + final source_url = WRITE_URL.replace('folder', 'source') final source = Paths.get(new URI(source_url)) Files.write(source, TEXT.bytes) and: @@ -471,7 +471,7 @@ class QuiltNioTest extends QuiltSpecification { thrown(FileSystemException) } - @Ignore + @Ignore('Can not write to null_path') void 'should stream directory content'() { given: makeObject(null_path('foo/file1.txt'), 'A') @@ -515,7 +515,7 @@ class QuiltNioTest extends QuiltSpecification { list == [ 'file4.txt' ] } - @Ignore + @Ignore('Can not write to null_path') void 'should check walkTree'() { given: makeObject(null_path('foo/file1.txt'), 'A')