Skip to content

Commit

Permalink
Feature: Parallel Building (#3476)
Browse files Browse the repository at this point in the history 
* perf: process smali code in parallel

Note: backsmali can't be properly multithreaded because of the synchronized methods inside

* perf: start backsmali concurrently with a resources decompiler

* perf: speed up apk building by skipping temp archive creation

Now we're not compressing the same data twice

* refactor: extract duplicated code

* refactor: rename methods and inline some comments
  • Loading branch information
@Cregrant
Cregrant authored Dec 26, 2023
1 parent 0741664 commit 81aae69
Show file tree
Hide file tree
Showing 4 changed files with 212 additions and 149 deletions.
203 changes: 87 additions & 116 deletions brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,17 +42,19 @@
import java.io.*;
import java.nio.file.Files;
import java.util.*;
import java.util.concurrent.atomic.AtomicReference;
import java.util.logging.Logger;
import java.util.zip.CRC32;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipOutputStream;

public class ApkBuilder {
private final static Logger LOGGER = Logger.getLogger(ApkBuilder.class.getName());

private final AtomicReference<AndrolibException> mBuildError = new AtomicReference<>(null);
private final Config mConfig;
private final ExtFile mApkDir;
private BackgroundWorker mWorker;
private ApkInfo mApkInfo;
private int mMinSdkVersion = 0;

Expand All @@ -78,51 +80,55 @@ public ApkBuilder(Config config, ExtFile apkDir) {

public void build(File outFile) throws BrutException {
LOGGER.info("Using Apktool " + ApktoolProperties.getVersion());
try {
mWorker = new BackgroundWorker();
mApkInfo = ApkInfo.load(mApkDir);

mApkInfo = ApkInfo.load(mApkDir);
if (mApkInfo.getSdkInfo() != null && mApkInfo.getSdkInfo().get("minSdkVersion") != null) {
String minSdkVersion = mApkInfo.getSdkInfo().get("minSdkVersion");
mMinSdkVersion = mApkInfo.getMinSdkVersionFromAndroidCodename(minSdkVersion);
}

if (mApkInfo.getSdkInfo() != null && mApkInfo.getSdkInfo().get("minSdkVersion") != null) {
String minSdkVersion = mApkInfo.getSdkInfo().get("minSdkVersion");
mMinSdkVersion = mApkInfo.getMinSdkVersionFromAndroidCodename(minSdkVersion);
}
if (outFile == null) {
String outFileName = mApkInfo.apkFileName;
outFile = new File(mApkDir, "dist" + File.separator + (outFileName == null ? "out.apk" : outFileName));
}

if (outFile == null) {
String outFileName = mApkInfo.apkFileName;
outFile = new File(mApkDir, "dist" + File.separator + (outFileName == null ? "out.apk" : outFileName));
}
//noinspection ResultOfMethodCallIgnored
new File(mApkDir, APK_DIRNAME).mkdirs();
File manifest = new File(mApkDir, "AndroidManifest.xml");
File manifestOriginal = new File(mApkDir, "AndroidManifest.xml.orig");

scheduleBuildDexFiles();
backupManifestFile(manifest, manifestOriginal);
buildResources();
copyLibs();
copyOriginalFilesIfEnabled();
mWorker.waitForFinish();
if (mBuildError.get() != null) {
throw mBuildError.get();
}

//noinspection ResultOfMethodCallIgnored
new File(mApkDir, APK_DIRNAME).mkdirs();
File manifest = new File(mApkDir, "AndroidManifest.xml");
File manifestOriginal = new File(mApkDir, "AndroidManifest.xml.orig");

buildSources();
buildNonDefaultSources();
buildManifestFile(manifest, manifestOriginal);
buildResources();
buildLibs();
buildCopyOriginalFiles();
buildApk(outFile);

// we must go after the Apk is built, and copy the files in via Zip
// this is because Aapt won't add files it doesn't know (ex unknown files)
buildUnknownFiles(outFile);

// we copied the AndroidManifest.xml to AndroidManifest.xml.orig so we can edit it
// lets restore the unedited one, to not change the original
if (manifest.isFile() && manifest.exists() && manifestOriginal.isFile()) {
try {
if (new File(mApkDir, "AndroidManifest.xml").delete()) {
FileUtils.moveFile(manifestOriginal, manifest);
buildApk(outFile);

// we copied the AndroidManifest.xml to AndroidManifest.xml.orig so we can edit it
// lets restore the unedited one, to not change the original
if (manifest.isFile() && manifest.exists() && manifestOriginal.isFile()) {
try {
if (new File(mApkDir, "AndroidManifest.xml").delete()) {
FileUtils.moveFile(manifestOriginal, manifest);
}
} catch (IOException ex) {
throw new AndrolibException(ex.getMessage());
}
} catch (IOException ex) {
throw new AndrolibException(ex.getMessage());
}
LOGGER.info("Built apk into: " + outFile.getPath());
} finally {
mWorker.shutdownNow();
}
LOGGER.info("Built apk into: " + outFile.getPath());
}

private void buildManifestFile(File manifest, File manifestOriginal) throws AndrolibException {
private void backupManifestFile(File manifest, File manifestOriginal) throws AndrolibException {
// If we decoded in "raw", we cannot patch AndroidManifest
if (new File(mApkDir, "resources.arsc").exists()) {
return;
Expand All @@ -141,24 +147,17 @@ private void buildManifestFile(File manifest, File manifestOriginal) throws Andr
}
}

private void buildSources() throws AndrolibException {
if (!buildSourcesRaw("classes.dex") && !buildSourcesSmali("smali", "classes.dex")) {
LOGGER.warning("Could not find sources");
}
}

private void buildNonDefaultSources() throws AndrolibException {
private void scheduleBuildDexFiles() throws AndrolibException {
try {
mWorker.submit(() -> scheduleDexBuild("classes.dex", "smali"));

// loop through any smali_ directories for multi-dex apks
Map<String, Directory> dirs = mApkDir.getDirectory().getDirs();
for (Map.Entry<String, Directory> directory : dirs.entrySet()) {
String name = directory.getKey();
if (name.startsWith("smali_")) {
String filename = name.substring(name.indexOf("_") + 1) + ".dex";

if (!buildSourcesRaw(filename) && !buildSourcesSmali(name, filename)) {
LOGGER.warning("Could not find sources");
}
mWorker.submit(() -> scheduleDexBuild(filename, name));
}
}

Expand All @@ -177,6 +176,19 @@ private void buildNonDefaultSources() throws AndrolibException {
}
}

private void scheduleDexBuild(String filename, String smali) {
try {
if (mBuildError.get() != null) {
return;
}
if (!buildSourcesRaw(filename) && !buildSourcesSmali(smali, filename)) {
LOGGER.warning("Could not find sources");
}
} catch (AndrolibException e) {
mBuildError.compareAndSet(null, e);
}
}

private boolean buildSourcesRaw(String filename) throws AndrolibException {
File working = new File(mApkDir, filename);
if (!working.exists()) {
Expand Down Expand Up @@ -214,6 +226,7 @@ private boolean buildSourcesSmali(String folder, String filename) throws Androli
}

private void buildResources() throws BrutException {
// create res folder, manifest file and resources.arsc
if (!buildResourcesRaw() && !buildResourcesFull() && !buildManifest()) {
LOGGER.warning("Could not find resources");
}
Expand Down Expand Up @@ -375,7 +388,7 @@ private boolean buildManifest() throws BrutException {
}
}

private void buildLibs() throws AndrolibException {
private void copyLibs() throws AndrolibException {
buildLibrary("lib");
buildLibrary("libs");
buildLibrary("kotlin");
Expand All @@ -401,7 +414,7 @@ private void buildLibrary(String folder) throws AndrolibException {
}
}

private void buildCopyOriginalFiles() throws AndrolibException {
private void copyOriginalFilesIfEnabled() throws AndrolibException {
if (mConfig.copyOriginalFiles) {
File originalDir = new File(mApkDir, "original");
if (originalDir.exists()) {
Expand All @@ -427,49 +440,34 @@ private void buildCopyOriginalFiles() throws AndrolibException {
}
}

private void buildUnknownFiles(File outFile) throws AndrolibException {
if (mApkInfo.unknownFiles != null) {
LOGGER.info("Copying unknown files/dir...");

Map<String, String> files = mApkInfo.unknownFiles;
File tempFile = new File(outFile.getParent(), outFile.getName() + ".apktool_temp");
boolean renamed = outFile.renameTo(tempFile);
if (!renamed) {
throw new AndrolibException("Unable to rename temporary file");
}

try (
ZipFile inputFile = new ZipFile(tempFile);
ZipOutputStream actualOutput = new ZipOutputStream(Files.newOutputStream(outFile.toPath()))
) {
copyExistingFiles(inputFile, actualOutput);
copyUnknownFiles(actualOutput, files);
} catch (IOException | BrutException ex) {
throw new AndrolibException(ex);
}

// Remove our temporary file.
private void buildApk(File outApk) throws AndrolibException {
LOGGER.info("Building apk file...");
if (outApk.exists()) {
//noinspection ResultOfMethodCallIgnored
tempFile.delete();
outApk.delete();
} else {
File outDir = outApk.getParentFile();
if (outDir != null && !outDir.exists()) {
//noinspection ResultOfMethodCallIgnored
outDir.mkdirs();
}
}
}

private void copyExistingFiles(ZipFile inputFile, ZipOutputStream outputFile) throws IOException {
// First, copy the contents from the existing outFile:
Enumeration<? extends ZipEntry> entries = inputFile.entries();
while (entries.hasMoreElements()) {
ZipEntry entry = new ZipEntry(entries.nextElement());

// We can't reuse the compressed size because it depends on compression sizes.
entry.setCompressedSize(-1);
outputFile.putNextEntry(entry);
File assetDir = new File(mApkDir, "assets");
if (!assetDir.exists()) {
assetDir = null;
}
try (ZipOutputStream zipOutputStream = new ZipOutputStream(Files.newOutputStream(outApk.toPath()))) {
// zip all AAPT-generated files
ZipUtils.zipFoldersPreserveStream(new File(mApkDir, APK_DIRNAME), zipOutputStream, assetDir, mApkInfo.doNotCompress);

// No need to create directory entries in the final apk
if (!entry.isDirectory()) {
BrutIO.copy(inputFile, outputFile, entry);
// we must copy some files manually
// this is because Aapt won't add files it doesn't know (ex unknown files)
if (mApkInfo.unknownFiles != null) {
LOGGER.info("Copying unknown files/dir...");
copyUnknownFiles(zipOutputStream, mApkInfo.unknownFiles);
}

outputFile.closeEntry();
} catch (IOException | BrutException e) {
throw new AndrolibException(e);
}
}

Expand Down Expand Up @@ -513,33 +511,6 @@ private void copyUnknownFiles(ZipOutputStream outputFile, Map<String, String> fi
}
}

private void buildApk(File outApk) throws AndrolibException {
LOGGER.info("Building apk file...");
if (outApk.exists()) {
//noinspection ResultOfMethodCallIgnored
outApk.delete();
} else {
File outDir = outApk.getParentFile();
if (outDir != null && !outDir.exists()) {
//noinspection ResultOfMethodCallIgnored
outDir.mkdirs();
}
}
File assetDir = new File(mApkDir, "assets");
if (!assetDir.exists()) {
assetDir = null;
}
zipPackage(outApk, new File(mApkDir, APK_DIRNAME), assetDir);
}

private void zipPackage(File apkFile, File rawDir, File assetDir) throws AndrolibException {
try {
ZipUtils.zipFolders(rawDir, apkFile, assetDir, mApkInfo.doNotCompress);
} catch (IOException | BrutException ex) {
throw new AndrolibException(ex);
}
}

private File[] getIncludeFiles() throws AndrolibException {
UsesFramework usesFramework = mApkInfo.usesFramework;
if (usesFramework == null) {
Expand Down
Loading

0 comments on commit 81aae69

Please sign in to comment.