Skip to content

Commit

Permalink
* Add new Builder.configDirectory option for Generator to output…
Browse files Browse the repository at this point in the history 
… files that GraalVM needs for AOT compilation (issue deeplearning4j/deeplearning4j#7362)
  • Loading branch information
@saudet
saudet committed Jun 8, 2020
1 parent 6d84ea7 commit c777f40
Show file tree
Hide file tree
Showing 5 changed files with 153 additions and 8 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@

* Add new `Builder.configDirectory` option for `Generator` to output files that GraalVM needs for AOT compilation ([issue eclipse/deeplearning4j#7362](https://github.com/eclipse/deeplearning4j/issues/7362))
* Fix `Parser` error on `template<>` containing non-type parameters without names ([issue bytedeco/javacpp-presets#889](https://github.com/bytedeco/javacpp-presets/issues/889))
* Bundle also the `vcruntime140_1.dll` and `msvcp140_1.dll` redist files from Visual Studio
* Fix `Builder` for different "java.home" path returned by latest JDKs from Oracle ([pull #400](https://github.com/bytedeco/javacpp/pull/400))
Expand Down
4 changes: 4 additions & 0 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,7 @@
<exclude>${javacpp.platform.library.path}/</exclude>
<exclude>${javacpp.platform}${javacpp.platform.extension}/</exclude>
<exclude>org/bytedeco/javacpp/${javacpp.platform}${javacpp.platform.extension}/</exclude>
<exclude>META-INF/native-image/</exclude>
<exclude>META-INF/MANIFEST.MF-*</exclude>
</excludes>
</configuration>
Expand Down Expand Up @@ -433,6 +434,8 @@ Import-Package: \
<argument>-classpath</argument>
<argument>${project.build.outputDirectory}</argument>
<argument>-copylibs</argument>
<argument>-configdir</argument>
<argument>${project.build.outputDirectory}/META-INF/native-image/${javacpp.platform}${javacpp.platform.extension}/</argument>
<argument>-properties</argument>
<argument>${javacpp.platform}${javacpp.platform.suffix}</argument>
<argument>-Dplatform.root=${javacpp.platform.root}</argument>
Expand Down Expand Up @@ -465,6 +468,7 @@ Import-Package: \
<include>${javacpp.platform.library.path}/</include>
<include>${javacpp.platform}${javacpp.platform.extension}/</include>
<include>org/bytedeco/javacpp/${javacpp.platform}${javacpp.platform.extension}/</include>
<include>META-INF/native-image/${javacpp.platform}${javacpp.platform.extension}/</include>
</includes>
<excludes>
<exclude>org/bytedeco/javacpp/windows-*/*.exp</exclude>
Expand Down
6 changes: 6 additions & 0 deletions src/main/java/org/bytedeco/javacpp/tools/BuildMojo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,10 @@ public class BuildMojo extends AbstractMojo {
@Parameter(property = "javacpp.copyResources", defaultValue = "false")
boolean copyResources = false;

/** Also create config files for GraalVM native-image in directory. */
@Parameter(property = "javacpp.configDirectory")
String configDirectory = null;

/** Also create a JAR file named {@code <jarPrefix>-<platform>.jar}. */
@Parameter(property = "javacpp.jarPrefix")
String jarPrefix = null;
Expand Down Expand Up @@ -280,6 +284,7 @@ String[] merge(String[] ss, String s) {
log.debug("header: " + header);
log.debug("copyLibs: " + copyLibs);
log.debug("copyResources: " + copyResources);
log.debug("configDirectory: " + configDirectory);
log.debug("jarPrefix: " + jarPrefix);
log.debug("properties: " + properties);
log.debug("propertyFile: " + propertyFile);
Expand Down Expand Up @@ -328,6 +333,7 @@ String[] merge(String[] ss, String s) {
.header(header)
.copyLibs(copyLibs)
.copyResources(copyResources)
.configDirectory(configDirectory)
.jarPrefix(jarPrefix)
.properties(properties)
.propertyFile(propertyFile)
Expand Down
80 changes: 76 additions & 4 deletions src/main/java/org/bytedeco/javacpp/tools/Builder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
Expand Down Expand Up @@ -547,6 +548,8 @@ File[] generateAndCompile(Class[] classes, String outputName, boolean first, boo
Generator generator = new Generator(logger, properties, encoding);
String[] sourceFilenames = {sourcePrefixes[0] + "jnijavacpp" + sourceSuffix,
sourcePrefixes[1] + outputName + sourceSuffix};
String[] configDirectories = {configDirectory != null ? new File(configDirectory, "jnijavacpp").getPath() : null,
configDirectory != null ? new File(configDirectory, outputName).getPath() : null};
String[] headerFilenames = {null, header ? sourcePrefixes[1] + outputName + ".h" : null};
String[] loadSuffixes = {"_jnijavacpp", null};
String[] baseLoadSuffixes = {null, "_jnijavacpp"};
Expand All @@ -563,21 +566,26 @@ File[] generateAndCompile(Class[] classes, String outputName, boolean first, boo
if (outputName.equals("jnijavacpp")) {
// generate a single file if the user only wants "jnijavacpp"
sourceFilenames = new String[] {sourcePrefixes[0] + outputName + sourceSuffix};
configDirectories = new String[] {configDirectory != null ? new File(configDirectory, outputName).getPath() : null};
headerFilenames = new String[] {header ? sourcePrefixes[0] + outputName + ".h" : null};
loadSuffixes = new String[] {null};
baseLoadSuffixes = new String[] {null};
classPaths = new String[] {null};
classesArray = new Class[][] {null};
classPaths = new String[] {classPath};
classesArray = new Class[][] {classes};
libraryNames = new String[] {libraryPrefix + outputName + librarySuffix};
}

boolean generated = true;
String[] jniConfigFilenames = new String[sourceFilenames.length];
String[] reflectConfigFilenames = new String[sourceFilenames.length];
for (int i = 0; i < sourceFilenames.length; i++) {
if (i == 0 && !first) {
continue;
}
logger.info("Generating " + sourceFilenames[i]);
if (!generator.generate(sourceFilenames[i], headerFilenames[i],
jniConfigFilenames[i] = configDirectories[i] != null ? configDirectories[i] + File.separator + "jni-config.json" : null;
reflectConfigFilenames[i] = configDirectories[i] != null ? configDirectories[i] + File.separator + "reflect-config.json" : null;
if (!generator.generate(sourceFilenames[i], jniConfigFilenames[i], reflectConfigFilenames[i], headerFilenames[i],
loadSuffixes[i], baseLoadSuffixes[i], classPaths[i], classesArray[i])) {
logger.info("Nothing generated for " + sourceFilenames[i]);
generated = false;
Expand Down Expand Up @@ -625,6 +633,30 @@ File[] generateAndCompile(Class[] classes, String outputName, boolean first, boo
outputFiles[i] = new File(sourceFilenames[i]);
}
}

if (header) {
for (String headerFilename : headerFilenames) {
if (headerFilename != null) {
outputFiles = Arrays.copyOf(outputFiles, outputFiles.length + 1);
outputFiles[outputFiles.length - 1] = new File(headerFilename);
}
}
}

if (configDirectory != null) {
for (String jniConfigFilename : jniConfigFilenames) {
if (jniConfigFilename != null) {
outputFiles = Arrays.copyOf(outputFiles, outputFiles.length + 1);
outputFiles[outputFiles.length - 1] = new File(jniConfigFilename);
}
}
for (String reflectConfigFilename : reflectConfigFilenames) {
if (reflectConfigFilename != null) {
outputFiles = Arrays.copyOf(outputFiles, outputFiles.length + 1);
outputFiles[outputFiles.length - 1] = new File(reflectConfigFilename);
}
}
}
}
return outputFiles;
}
Expand Down Expand Up @@ -706,6 +738,8 @@ public Builder(Logger logger) {
/** The name of the output generated source file or shared library. This enables single-
* file output mode. By default, the top-level enclosing classes get one file each. */
String outputName = null;
/** The name of the directory where to output config files for GraalVM native-image, if not {@code null}. */
File configDirectory = null;
/** The name of the JAR file to create, if not {@code null}. */
String jarPrefix = null;
/** If true, deletes all files from {@link #outputDirectory} before writing anything in it. */
Expand Down Expand Up @@ -802,6 +836,16 @@ public Builder outputName(String outputName) {
this.outputName = outputName;
return this;
}
/** Sets the {@link #configDirectory} field to the argument. */
public Builder configDirectory(String configDirectory) {
configDirectory(configDirectory == null ? null : new File(configDirectory));
return this;
}
/** Sets the {@link #configDirectory} field to the argument. */
public Builder configDirectory(File configDirectory) {
this.configDirectory = configDirectory;
return this;
}
/** Sets the {@link #jarPrefix} field to the argument. */
public Builder jarPrefix(String jarPrefix) {
this.jarPrefix = jarPrefix;
Expand Down Expand Up @@ -1128,7 +1172,7 @@ public File[] build() throws IOException, InterruptedException, ParserException

if (files != null && files.length > 0) {
// files[0] might be null if "jnijavacpp" was not generated and compiled
File directory = files[files.length - 1].getParentFile();
File directory = files[0] != null ? files[0].getParentFile() : files[files.length - 1].getParentFile();
outputFiles.addAll(Arrays.asList(files));
if (copyLibs) {
// Do not copy library files from inherit properties ...
Expand Down Expand Up @@ -1163,6 +1207,8 @@ public File[] build() throws IOException, InterruptedException, ParserException
logger.info("Copying " + fi);
Files.copy(fi.toPath(), fo.toPath(), StandardCopyOption.REPLACE_EXISTING);
outputFiles.add(fo);
files = Arrays.copyOf(files, files.length + 1);
files[files.length - 1] = fo;
}
}
}
Expand Down Expand Up @@ -1206,6 +1252,29 @@ public File[] build() throws IOException, InterruptedException, ParserException
}
}
}
if (configDirectory != null) {
File file = new File(configDirectory, name + "/resource-config.json");
File dir = file.getParentFile();
if (dir != null) {
dir.mkdirs();
}
logger.info("Generating " + file);
try (PrintWriter out = encoding != null ? new PrintWriter(file, encoding) : new PrintWriter(file)) {
out.println("{");
out.println(" \"resources\": [");
out.print(" {\"pattern\": \"META-INF/.*\"}");
String separator = "," + System.lineSeparator();
for (File f : files != null ? files : new File[0]) {
if (!f.toPath().startsWith(configDirectory.toPath())) {
out.print(separator + " {\"pattern\": \".*/" + f.getName() + "\"}");
}
}
out.println();
out.println(" ]");
out.println("}");
}
outputFiles.add(file);
}
}

File[] files = outputFiles.toArray(new File[outputFiles.size()]);
Expand Down Expand Up @@ -1251,6 +1320,7 @@ public static void printHelp() {
System.out.println(" -header Generate header file with declarations of callbacks functions");
System.out.println(" -copylibs Copy to output directory dependent libraries (link and preload)");
System.out.println(" -copyresources Copy to output directory resources listed in properties");
System.out.println(" -configdir <directory> Also create config files for GraalVM native-image in directory");
System.out.println(" -jarprefix <prefix> Also create a JAR file named \"<prefix>-<platform>.jar\"");
System.out.println(" -properties <resource> Load all platform properties from resource");
System.out.println(" -propertyfile <file> Load all platform properties from file");
Expand Down Expand Up @@ -1304,6 +1374,8 @@ public static void main(String[] args) throws Exception {
builder.copyLibs(true);
} else if ("-copyresources".equals(args[i])) {
builder.copyResources(true);
} else if ("-configdir".equals(args[i])) {
builder.configDirectory(args[++i]);
} else if ("-jarprefix".equals(args[i])) {
builder.jarPrefix(args[++i]);
} else if ("-properties".equals(args[i])) {
Expand Down
70 changes: 66 additions & 4 deletions src/main/java/org/bytedeco/javacpp/tools/Generator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,23 +170,23 @@ static enum LongEnum { LONG; long value; }
final Logger logger;
final Properties properties;
final String encoding;
PrintWriter out, out2;
PrintWriter out, out2, jniConfigOut, reflectConfigOut;
Map<String,String> callbacks;
IndexedSet<Class> functions, deallocators, arrayDeallocators, jclasses;
Map<Class,Set<String>> members, virtualFunctions, virtualMembers;
Map<Method,MethodInformation> annotationCache;
boolean mayThrowExceptions, usesAdapters, passesStrings, accessesEnums;

public boolean generate(String sourceFilename, String headerFilename, String loadSuffix,
String baseLoadSuffix, String classPath, Class<?> ... classes) throws IOException {
public boolean generate(String sourceFilename, String jniConfigFilename, String reflectConfigFilename, String headerFilename,
String loadSuffix, String baseLoadSuffix, String classPath, Class<?> ... classes) throws IOException {
try {
// first pass using a null writer to fill up the IndexedSet objects
out = new PrintWriter(new Writer() {
@Override public void write(char[] cbuf, int off, int len) { }
@Override public void flush() { }
@Override public void close() { }
});
out2 = null;
out2 = jniConfigOut = reflectConfigOut= null;
callbacks = new LinkedHashMap<String,String>();
functions = new IndexedSet<Class>();
deallocators = new IndexedSet<Class>();
Expand Down Expand Up @@ -221,6 +221,24 @@ public boolean generate(String sourceFilename, String headerFilename, String loa
}
out2 = encoding != null ? new PrintWriter(headerFile, encoding) : new PrintWriter(headerFile);
}
if (jniConfigFilename != null) {
logger.info("Generating " + jniConfigFilename);
File jniConfigFile = new File(jniConfigFilename);
File jniConfigDir = jniConfigFile.getParentFile();
if (jniConfigDir != null) {
jniConfigDir.mkdirs();
}
jniConfigOut = encoding != null ? new PrintWriter(jniConfigFile, encoding) : new PrintWriter(jniConfigFile);
}
if (reflectConfigFilename != null) {
logger.info("Generating " + reflectConfigFilename);
File reflectConfigFile = new File(reflectConfigFilename);
File reflectConfigDir = reflectConfigFile.getParentFile();
if (reflectConfigDir != null) {
reflectConfigDir.mkdirs();
}
reflectConfigOut = encoding != null ? new PrintWriter(reflectConfigFile, encoding) : new PrintWriter(reflectConfigFile);
}
return classes(mayThrowExceptions, usesAdapters, passesStrings, accessesEnums, loadSuffix, baseLoadSuffix, classPath, classes);
} else {
return false;
Expand All @@ -232,6 +250,12 @@ public boolean generate(String sourceFilename, String headerFilename, String loa
if (out2 != null) {
out2.close();
}
if (jniConfigOut != null) {
jniConfigOut.close();
}
if (reflectConfigOut != null) {
reflectConfigOut.close();
}
}
}

Expand Down Expand Up @@ -1709,6 +1733,44 @@ boolean classes(boolean handleExceptions, boolean defineAdapters, boolean conver
out2.println("#endif");
}

for (PrintWriter o : new PrintWriter[] {jniConfigOut, reflectConfigOut}) {
allClasses.addAll(jclasses.keySet());

LinkedHashSet<Class> reflectClasses = new LinkedHashSet<Class>();
reflectClasses.addAll(baseClasses);
reflectClasses.add(Object.class);
reflectClasses.add(Buffer.class);
reflectClasses.add(String.class);

if (o != null) {
o.println("[");
String separator = "";
for (Class cls : allClasses) {
do {
o.println(separator + " {");
o.print(" \"name\" : \"" + cls.getName() + "\"");
if (reflectClasses.contains(cls) || reflectClasses.contains(cls.getEnclosingClass())) {
o.println(",");
o.println(" \"allDeclaredConstructors\" : true,");
o.println(" \"allPublicConstructors\" : true,");
o.println(" \"allDeclaredMethods\" : true,");
o.println(" \"allPublicMethods\" : true,");
o.println(" \"allDeclaredFields\" : true,");
o.println(" \"allPublicFields\" : true,");
o.println(" \"allDeclaredClasses\" : true,");
o.print(" \"allPublicClasses\" : true");
}
o.println();
o.print(" }");
separator = "," + System.lineSeparator();
cls = cls.getEnclosingClass();
} while (cls != null);
}
o.println();
o.println("]");
}
}

return supportedPlatform;
}

Expand Down

0 comments on commit c777f40

Please sign in to comment.