diff --git a/src/main/java/ApplyParchmentToSourceJar.java b/src/main/java/ApplyParchmentToSourceJar.java index b378b8f..941f3b8 100644 --- a/src/main/java/ApplyParchmentToSourceJar.java +++ b/src/main/java/ApplyParchmentToSourceJar.java @@ -1,5 +1,4 @@ import com.intellij.core.CoreApplicationEnvironment; -import com.intellij.core.CoreJavaFileManager; import com.intellij.core.JavaCoreApplicationEnvironment; import com.intellij.core.JavaCoreProjectEnvironment; import com.intellij.lang.jvm.facade.JvmElementProvider; @@ -12,6 +11,7 @@ import com.intellij.openapi.util.Disposer; import com.intellij.openapi.util.registry.Registry; import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.openapi.vfs.VirtualFileSystem; import com.intellij.openapi.vfs.impl.ZipHandler; import com.intellij.pom.java.InternalPersistentJavaLanguageLevelReaderService; import com.intellij.pom.java.LanguageLevel; @@ -23,17 +23,19 @@ import com.intellij.psi.impl.PsiElementFinderImpl; import com.intellij.psi.impl.PsiNameHelperImpl; import com.intellij.psi.impl.PsiTreeChangePreprocessor; -import com.intellij.psi.impl.file.impl.JavaFileManager; import com.intellij.psi.impl.source.tree.JavaTreeGenerator; import com.intellij.psi.impl.source.tree.TreeGenerator; import com.intellij.psi.util.JavaClassSupers; +import modules.CoreJrtFileSystem; import namesanddocs.NameAndDocSourceLoader; import namesanddocs.NamesAndDocsDatabase; import org.jetbrains.annotations.NotNull; +import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.PrintStream; +import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -58,20 +60,28 @@ public class ApplyParchmentToSourceJar implements AutoCloseable { private boolean enableJavadoc = true; private final Disposable rootDisposable; - public ApplyParchmentToSourceJar(NamesAndDocsDatabase namesAndDocs) throws IOException { + public ApplyParchmentToSourceJar(Path javaHome, NamesAndDocsDatabase namesAndDocs) throws IOException { this.namesAndDocs = namesAndDocs; tempDir = Files.createTempDirectory("applyparchment"); this.rootDisposable = Disposer.newDisposable(); + System.setProperty("idea.home.path", tempDir.toAbsolutePath().toString()); // IDEA requires a config directory, even if it's empty PathManager.setExplicitConfigPath(tempDir.toAbsolutePath().toString()); Registry.markAsLoaded(); // Avoids warnings about config not being loaded - var appEnv = new JavaCoreApplicationEnvironment(rootDisposable); + var appEnv = new JavaCoreApplicationEnvironment(rootDisposable) { + @Override + protected VirtualFileSystem createJrtFileSystem() { + return new CoreJrtFileSystem(); + } + }; initAppExtensionsAndServices(appEnv); javaEnv = new JavaCoreProjectEnvironment(rootDisposable, appEnv); + ClasspathSetup.addJdkModules(javaHome, javaEnv); + project = javaEnv.getProject(); initProjectExtensionsAndServices(project); @@ -81,10 +91,11 @@ public ApplyParchmentToSourceJar(NamesAndDocsDatabase namesAndDocs) throws IOExc psiManager = PsiManager.getInstance(project); } + public static void main(String[] args) throws Exception { System.setProperty("java.awt.headless", "true"); - Path inputPath = null, outputPath = null, namesAndDocsPath = null; + Path inputPath = null, outputPath = null, namesAndDocsPath = null, librariesPath = null; boolean enableJavadoc = true; int queueDepth = 50; @@ -105,6 +116,13 @@ public static void main(String[] args) throws Exception { } outputPath = Paths.get(args[++i]); break; + case "--libraries": + if (i + 1 >= args.length) { + System.err.println("Missing argument for --libraries"); + System.exit(1); + } + librariesPath = Paths.get(args[++i]); + break; case "--names": if (i + 1 >= args.length) { System.err.println("Missing argument for --names"); @@ -142,7 +160,15 @@ public static void main(String[] args) throws Exception { var namesAndDocs = NameAndDocSourceLoader.load(namesAndDocsPath); - try (var applyParchment = new ApplyParchmentToSourceJar(namesAndDocs)) { + // Add the Java Runtime we are currently running in + var javaHome = Paths.get(System.getProperty("java.home")); + + try (var applyParchment = new ApplyParchmentToSourceJar(javaHome, namesAndDocs)) { + // Add external libraries to classpath + if (librariesPath != null) { + ClasspathSetup.addLibraries(librariesPath, applyParchment.javaEnv); + } + applyParchment.setMaxQueueDepth(queueDepth); applyParchment.setEnableJavadoc(enableJavadoc); applyParchment.apply(inputPath, outputPath); @@ -169,21 +195,6 @@ public void apply(Path inputPath, Path outputPath) throws IOException, Interrupt javaEnv.addSourcesToClasspath(sourceJarRoot); - var javaFileManager = (CoreJavaFileManager) JavaFileManager.getInstance(project); - javaFileManager.addToClasspath(sourceJarRoot); - -// Files.readAllLines(Paths.get(librariesPath)) -// .stream() -// .filter(l -> l.startsWith("-e=")) -// .map(l -> l.substring(3)) -// .map(File::new) -// .forEach(file -> { -// if (!file.exists()) { -// throw new UncheckedIOException(new FileNotFoundException(file.getAbsolutePath())); -// } -// javaEnv.addJarToClassPath(file); -// }); - try (var zin = new ZipInputStream(Files.newInputStream(inputPath)); var fout = Files.newOutputStream(outputPath); var asyncZout = new OrderedWorkQueue(new ZipOutputStream(fout), maxQueueDepth)) { diff --git a/src/main/java/ClasspathSetup.java b/src/main/java/ClasspathSetup.java new file mode 100644 index 0000000..45f8b01 --- /dev/null +++ b/src/main/java/ClasspathSetup.java @@ -0,0 +1,95 @@ +import com.intellij.core.JavaCoreProjectEnvironment; +import com.intellij.openapi.util.text.StringUtil; +import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.util.io.URLUtil; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Properties; + +public final class ClasspathSetup { + private ClasspathSetup() { + } + + public static void addJdkModules(Path jdkHome, JavaCoreProjectEnvironment javaEnv) { + var jrtFileSystem = javaEnv.getEnvironment().getJrtFileSystem(); + + VirtualFile jdkVfsRoot = jrtFileSystem.findFileByPath(jdkHome.toAbsolutePath() + URLUtil.JAR_SEPARATOR); + if (jdkVfsRoot == null) { + System.err.println("Failed to load VFS-entry for JDK home " + jdkHome + ". Is it missing?"); + return; + } + + var modulesFolder = jdkVfsRoot.findChild("modules"); + if (modulesFolder == null) { + System.err.println("VFS for JDK " + jdkHome + " doesn't have a modules subfolder"); + return; + } + + int moduleCount = 0; + List modules = readModulesFromReleaseFile(jdkHome); + if (modules != null) { + for (String module : modules) { + var moduleRoot = modulesFolder.findChild(module); + if (moduleRoot == null || !moduleRoot.isDirectory()) { + System.err.println("Couldn't find module " + module + " even though it was listed in the release file of JDK " + jdkHome); + } else { + javaEnv.addSourcesToClasspath(moduleRoot); + moduleCount++; + } + } + } else { + + for (VirtualFile jrtChild : modulesFolder.getChildren()) { + if (jrtChild.isDirectory()) { + javaEnv.addSourcesToClasspath(jrtChild); + moduleCount++; + } + } + } + + System.out.println("Added " + moduleCount + " modules from " + jdkHome); + } + + public static void addLibraries(Path librariesPath, JavaCoreProjectEnvironment javaEnv) throws IOException { + var libraryFiles = Files.readAllLines(librariesPath) + .stream() + .filter(l -> l.startsWith("-e=")) + .map(l -> l.substring(3)) + .map(File::new) + .toList(); + + for (var libraryFile : libraryFiles) { + if (!libraryFile.exists()) { + throw new UncheckedIOException(new FileNotFoundException(libraryFile.getAbsolutePath())); + } + javaEnv.addJarToClassPath(libraryFile); + System.out.println("Added " + libraryFile); + } + } + + /** + * Reads the "release" file found at the root of normal JDKs + */ + private static @Nullable List readModulesFromReleaseFile(@NotNull Path jrtBaseDir) { + try (InputStream stream = Files.newInputStream(jrtBaseDir.resolve("release"))) { + Properties p = new Properties(); + p.load(stream); + String modules = p.getProperty("MODULES"); + if (modules != null) { + return StringUtil.split(StringUtil.unquoteString(modules), " "); + } + } catch (IOException | IllegalArgumentException e) { + return null; + } + return null; + } +} diff --git a/src/main/java/GatherReplacementsVisitor.java b/src/main/java/GatherReplacementsVisitor.java index 9672fe4..87bb3b2 100644 --- a/src/main/java/GatherReplacementsVisitor.java +++ b/src/main/java/GatherReplacementsVisitor.java @@ -5,10 +5,14 @@ import com.intellij.psi.PsiElement; import com.intellij.psi.PsiField; import com.intellij.psi.PsiJavaDocumentedElement; +import com.intellij.psi.PsiLambdaExpression; import com.intellij.psi.PsiMethod; +import com.intellij.psi.PsiModifier; import com.intellij.psi.PsiParameter; import com.intellij.psi.PsiRecursiveElementVisitor; import com.intellij.psi.PsiReferenceExpression; +import com.intellij.psi.PsiType; +import com.intellij.psi.PsiTypes; import com.intellij.psi.PsiWhiteSpace; import com.intellij.psi.SyntaxTraverser; import com.intellij.psi.search.GlobalSearchScope; @@ -97,8 +101,13 @@ public void visitElement(@NotNull PsiElement element) { if (psiParameter.getNameIdentifier() == null) { continue; } - var paramData = methodData.getParameter(i); - if (paramData != null) { + + // Parchment stores parameter indices based on the index of the parameter in the actual compiled method + // to account for synthetic parameter not found in the source-code, we must adjust the index accordingly. + var jvmIndex = getJvmIndex(psiParameter, i); + + var paramData = methodData.getParameter(jvmIndex); + if (paramData != null && paramData.getName() != null) { // Replace parameters within the method body activeParameters.put(psiParameter, paramData); @@ -136,6 +145,28 @@ public void visitElement(@NotNull PsiElement element) { element.acceptChildren(this); } + private int getJvmIndex(PsiParameter psiParameter, int index) { + var declarationScope = psiParameter.getDeclarationScope(); + if (declarationScope instanceof PsiMethod psiMethod) { + + // Try to account for hidden parameters only present in bytecode since the + // mapping data refers to parameters using those indices + if (psiMethod.isConstructor() && psiMethod.getContainingClass() != null && psiMethod.getContainingClass().isEnum()) { + index += 2; + } else if (psiMethod.getContainingClass() != null && psiMethod.getContainingClass() != null + && !psiMethod.getContainingClass().hasModifierProperty(PsiModifier.STATIC)) { + index += 1; + } + + return index; + } else if (declarationScope instanceof PsiLambdaExpression psiLambda) { + // Naming lambdas doesn't really work + return index; + } else { + return -1; + } + } + private void applyJavadoc(PsiJavaDocumentedElement method, List javadoc, List replacements) { if (!enableJavadoc) { return; @@ -224,7 +255,7 @@ private NamesAndDocsForMethod getMethodData(@Nullable PsiMethod psiMethod) { var classData = getClassData(psiMethod.getContainingClass()); if (classData != null) { var methodName = psiMethod.getName(); - var methodSignature = ClassUtil.getAsmMethodSignature(psiMethod); + var methodSignature = getBinaryMethodSignature(psiMethod); methodData = Optional.ofNullable(classData.getMethod(methodName, methodSignature)); } @@ -233,6 +264,29 @@ private NamesAndDocsForMethod getMethodData(@Nullable PsiMethod psiMethod) { } } + public static String getBinaryMethodSignature(PsiMethod method) { + StringBuilder signature = new StringBuilder(); + signature.append("("); + for (PsiParameter param : method.getParameterList().getParameters()) { + var binaryPresentation = ClassUtil.getBinaryPresentation(param.getType()); + if (binaryPresentation.isEmpty()) { + System.err.println("Failed to create binary representation for type " + param.getType().getCanonicalText()); + binaryPresentation = "ERROR"; + } + signature.append(binaryPresentation); + } + signature.append(")"); + var returnType = Optional.ofNullable(method.getReturnType()).orElse(PsiTypes.voidType()); + var returnTypeRepresentation = ClassUtil.getBinaryPresentation(returnType); + if (returnTypeRepresentation.isEmpty()) { + System.err.println("Failed to create binary representation for type " + returnType.getCanonicalText()); + returnTypeRepresentation = "ERROR"; + } + signature.append(returnTypeRepresentation); + return signature.toString(); + } + + /** * An adapted version of {@link ClassUtil#formatClassName(PsiClass, StringBuilder)} where Inner-Classes * use a $ separator while formatClassName separates InnerClasses with periods from their parent. diff --git a/src/main/java/modules/CoreJrtFileSystem.java b/src/main/java/modules/CoreJrtFileSystem.java new file mode 100644 index 0000000..6358d9b --- /dev/null +++ b/src/main/java/modules/CoreJrtFileSystem.java @@ -0,0 +1,99 @@ +package modules; + +import com.intellij.openapi.vfs.DeprecatedVirtualFileSystem; +import com.intellij.openapi.vfs.StandardFileSystems; +import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.util.containers.ConcurrentFactoryMap; +import com.intellij.util.io.URLUtil; +import org.jetbrains.annotations.NonNls; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URI; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.util.Map; + +public class CoreJrtFileSystem extends DeprecatedVirtualFileSystem { + + private final Map roots = ConcurrentFactoryMap.createMap(jdkHomePath -> { + var jdkHome = new File(jdkHomePath); + var jrtFsJar = getJrtFsJar(jdkHome); + if (!jrtFsJar.exists()) { + return null; + } + var rootUri = URI.create(StandardFileSystems.JRT_PROTOCOL + ":/"); + /* + The ClassLoader, that was used to load JRT FS Provider actually lives as long as current thread due to ThreadLocal leak in jrt-fs, + See https://bugs.openjdk.java.net/browse/JDK-8260621 + So that cache allows us to avoid creating too many classloaders for same JDK and reduce severity of that leak + */ + // If the runtime JDK is set to 9+ it has JrtFileSystemProvider, + // but to load proper jrt-fs (one that is pointed by jdkHome) we should provide "java.home" path + FileSystem fileSystem; + try { + fileSystem = FileSystems.newFileSystem(rootUri, Map.of("java.home", jdkHome.getAbsolutePath())); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + return new CoreJrtVirtualFile(this, jdkHomePath, fileSystem.getPath(""), null); + }); + + @Override + public @NonNls @NotNull String getProtocol() { + return StandardFileSystems.JRT_PROTOCOL; + } + + @Override + public @Nullable VirtualFile findFileByPath(@NotNull @NonNls String path) { + var splitPath = splitPath(path); + var jdkHomePath = splitPath.jdkHome; + var pathInImage = splitPath.pathInImage; + var root = roots.get(jdkHomePath); + if (root == null) { + return null; + } + + if (pathInImage.isEmpty()) return root; + + return root.findFileByRelativePath(pathInImage); + } + + @Override + public void refresh(boolean asynchronous) { + } + + @Override + public @Nullable VirtualFile refreshAndFindFileByPath(@NotNull String path) { + return findFileByPath(path); + } + + private void clearRoots() { + roots.clear(); + } + + private static File getJrtFsJar(File jdkHome) { + return new File(jdkHome, "lib/jrt-fs.jar"); + } + + static boolean isModularJdk(File jdkHome) { + return getJrtFsJar(jdkHome).exists(); + } + + private static JdkImagePath splitPath(String path) { + var separator = path.indexOf(URLUtil.JAR_SEPARATOR); + if (separator < 0) { + throw new IllegalArgumentException("Path in CoreJrtFileSystem must contain a separator: " + path); + } + var localPath = path.substring(0, separator); + var pathInJar = path.substring(separator + URLUtil.JAR_SEPARATOR.length()); + return new JdkImagePath(localPath, pathInJar); + } + + record JdkImagePath(String jdkHome, String pathInImage) { + } +} diff --git a/src/main/java/modules/CoreJrtVirtualFile.java b/src/main/java/modules/CoreJrtVirtualFile.java new file mode 100644 index 0000000..2f6dea1 --- /dev/null +++ b/src/main/java/modules/CoreJrtVirtualFile.java @@ -0,0 +1,170 @@ +package modules; + +import com.intellij.openapi.util.io.FileUtil; +import com.intellij.openapi.vfs.VfsUtilCore; +import com.intellij.openapi.vfs.VirtualFile; +import com.intellij.openapi.vfs.VirtualFileSystem; +import com.intellij.util.io.URLUtil; +import org.jetbrains.annotations.NonNls; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +public class CoreJrtVirtualFile extends VirtualFile { + + private final CoreJrtFileSystem virtualFileSystem; + private final String jdkHomePath; + private final Path path; + private final CoreJrtVirtualFile parent; + + public CoreJrtVirtualFile(CoreJrtFileSystem virtualFileSystem, String jdkHomePath, Path path, CoreJrtVirtualFile parent) { + this.virtualFileSystem = virtualFileSystem; + this.jdkHomePath = jdkHomePath; + this.path = path; + this.parent = parent; + } + + private BasicFileAttributes getAttributes() { + try { + return Files.readAttributes(path, BasicFileAttributes.class); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public @NotNull VirtualFileSystem getFileSystem() { + return virtualFileSystem; + } + + @Override + public @NotNull String getName() { + return path.getFileName().toString(); + } + + @Override + public @NonNls @NotNull String getPath() { + return FileUtil.toSystemIndependentName(jdkHomePath + URLUtil.JAR_SEPARATOR + path); + } + + @Override + public boolean isWritable() { + return false; + } + + @Override + public boolean isDirectory() { + return Files.isDirectory(path); + } + + @Override + public boolean isValid() { + return true; + } + + @Override + public com.intellij.openapi.vfs.VirtualFile getParent() { + return parent; + } + + @Nullable + private VirtualFile[] myChildren = null; + + private final ReadWriteLock rwl = new ReentrantReadWriteLock(); + + @Override + public com.intellij.openapi.vfs.VirtualFile[] getChildren() { + rwl.readLock().lock(); + try { + if (myChildren != null) { + return myChildren; + } + } finally { + rwl.readLock().unlock(); + } + + rwl.writeLock().lock(); + try { + if (myChildren == null) { + myChildren = computeChildren(); + } + return myChildren; + } finally { + rwl.writeLock().unlock(); + } + } + + private VirtualFile[] computeChildren() { + List paths = new ArrayList<>(); + try (var dirStream = Files.newDirectoryStream(path)) { + for (Path childPath : dirStream) { + paths.add(new CoreJrtVirtualFile(virtualFileSystem, jdkHomePath, childPath, this)); + } + } catch (IOException ignored) { + } + + if (paths.isEmpty()) { + return EMPTY_ARRAY; + } else { + return paths.toArray(new VirtualFile[0]); + } + } + + @Override + public @NotNull OutputStream getOutputStream(Object requestor, long newModificationStamp, long newTimeStamp) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public byte @NotNull [] contentsToByteArray() throws IOException { + return Files.readAllBytes(path); + } + + @Override + public long getTimeStamp() { + return getAttributes().lastModifiedTime().toMillis(); + } + + @Override + public long getLength() { + return getAttributes().size(); + } + + @Override + public void refresh(boolean asynchronous, boolean recursive, @Nullable Runnable postRunnable) { + } + + @Override + public @NotNull InputStream getInputStream() throws IOException { + return VfsUtilCore.inputStreamSkippingBOM(new BufferedInputStream(Files.newInputStream(path)), this); + } + + @Override + public long getModificationStamp() { + return 0; + } + + @Override + public boolean equals(Object obj) { + return obj instanceof CoreJrtVirtualFile jrtVf && + path == jrtVf.path + && virtualFileSystem == jrtVf.virtualFileSystem; + } + + @Override + public int hashCode() { + return path.hashCode(); + } +} diff --git a/src/test/java/Tests.java b/src/test/java/Tests.java index a9c02ce..0311df9 100644 --- a/src/test/java/Tests.java +++ b/src/test/java/Tests.java @@ -30,8 +30,11 @@ void testNesting() throws Exception { }, null); } + // Add the Java Runtime we are currently running in + var javaHome = Paths.get(System.getProperty("java.home")); + var ouptutFile = tempDir.resolve("output.jar"); - try (var remapper = new ApplyParchmentToSourceJar(NameAndDocSourceLoader.load(parchmentFile))) { + try (var remapper = new ApplyParchmentToSourceJar(javaHome, NameAndDocSourceLoader.load(parchmentFile))) { remapper.setMaxQueueDepth(0); // Easier to debug... remapper.apply(inputFile, ouptutFile); }