Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Feature: Parallel Building #3476

Merged
merged 5 commits into from
Dec 26, 2023
Merged

Feature: Parallel Building #3476

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4f7291b
perf: process smali code in parallel
Cregrant Dec 19, 2023
390a8fd
perf: start backsmali concurrently with a resources decompiler
Cregrant Dec 19, 2023
f410812
perf: speed up apk building by skipping temp archive creation
Cregrant Dec 20, 2023
d9db843
refactor: extract duplicated code
Cregrant Dec 21, 2023
9650d82
refactor: rename methods and inline some comments
Cregrant Dec 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
203 changes: 87 additions & 116 deletions brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,17 +42,19 @@
import java.io.*;
import java.nio.file.Files;
import java.util.*;
import java.util.concurrent.atomic.AtomicReference;
import java.util.logging.Logger;
import java.util.zip.CRC32;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipOutputStream;

public class ApkBuilder {
private final static Logger LOGGER = Logger.getLogger(ApkBuilder.class.getName());

private final AtomicReference<AndrolibException> mBuildError = new AtomicReference<>(null);
private final Config mConfig;
private final ExtFile mApkDir;
private BackgroundWorker mWorker;
private ApkInfo mApkInfo;
private int mMinSdkVersion = 0;

Expand All @@ -78,51 +80,55 @@ public ApkBuilder(Config config, ExtFile apkDir) {

public void build(File outFile) throws BrutException {
LOGGER.info("Using Apktool " + ApktoolProperties.getVersion());
try {
mWorker = new BackgroundWorker();
mApkInfo = ApkInfo.load(mApkDir);

mApkInfo = ApkInfo.load(mApkDir);
if (mApkInfo.getSdkInfo() != null && mApkInfo.getSdkInfo().get("minSdkVersion") != null) {
String minSdkVersion = mApkInfo.getSdkInfo().get("minSdkVersion");
mMinSdkVersion = mApkInfo.getMinSdkVersionFromAndroidCodename(minSdkVersion);
}

if (mApkInfo.getSdkInfo() != null && mApkInfo.getSdkInfo().get("minSdkVersion") != null) {
String minSdkVersion = mApkInfo.getSdkInfo().get("minSdkVersion");
mMinSdkVersion = mApkInfo.getMinSdkVersionFromAndroidCodename(minSdkVersion);
}
if (outFile == null) {
String outFileName = mApkInfo.apkFileName;
outFile = new File(mApkDir, "dist" + File.separator + (outFileName == null ? "out.apk" : outFileName));
}

if (outFile == null) {
String outFileName = mApkInfo.apkFileName;
outFile = new File(mApkDir, "dist" + File.separator + (outFileName == null ? "out.apk" : outFileName));
}
//noinspection ResultOfMethodCallIgnored
new File(mApkDir, APK_DIRNAME).mkdirs();
File manifest = new File(mApkDir, "AndroidManifest.xml");
File manifestOriginal = new File(mApkDir, "AndroidManifest.xml.orig");

scheduleBuildDexFiles();
backupManifestFile(manifest, manifestOriginal);
buildResources();
copyLibs();
copyOriginalFilesIfEnabled();
mWorker.waitForFinish();
if (mBuildError.get() != null) {
throw mBuildError.get();
}

//noinspection ResultOfMethodCallIgnored
new File(mApkDir, APK_DIRNAME).mkdirs();
File manifest = new File(mApkDir, "AndroidManifest.xml");
File manifestOriginal = new File(mApkDir, "AndroidManifest.xml.orig");

buildSources();
buildNonDefaultSources();
buildManifestFile(manifest, manifestOriginal);
buildResources();
buildLibs();
buildCopyOriginalFiles();
buildApk(outFile);

// we must go after the Apk is built, and copy the files in via Zip
// this is because Aapt won't add files it doesn't know (ex unknown files)
buildUnknownFiles(outFile);

// we copied the AndroidManifest.xml to AndroidManifest.xml.orig so we can edit it
// lets restore the unedited one, to not change the original
if (manifest.isFile() && manifest.exists() && manifestOriginal.isFile()) {
try {
if (new File(mApkDir, "AndroidManifest.xml").delete()) {
FileUtils.moveFile(manifestOriginal, manifest);
buildApk(outFile);

// we copied the AndroidManifest.xml to AndroidManifest.xml.orig so we can edit it
// lets restore the unedited one, to not change the original
if (manifest.isFile() && manifest.exists() && manifestOriginal.isFile()) {
try {
if (new File(mApkDir, "AndroidManifest.xml").delete()) {
FileUtils.moveFile(manifestOriginal, manifest);
}
} catch (IOException ex) {
throw new AndrolibException(ex.getMessage());
}
} catch (IOException ex) {
throw new AndrolibException(ex.getMessage());
}
LOGGER.info("Built apk into: " + outFile.getPath());
} finally {
mWorker.shutdownNow();
}
LOGGER.info("Built apk into: " + outFile.getPath());
}

private void buildManifestFile(File manifest, File manifestOriginal) throws AndrolibException {
private void backupManifestFile(File manifest, File manifestOriginal) throws AndrolibException {
// If we decoded in "raw", we cannot patch AndroidManifest
if (new File(mApkDir, "resources.arsc").exists()) {
return;
Expand All @@ -141,24 +147,17 @@ private void buildManifestFile(File manifest, File manifestOriginal) throws Andr
}
}

private void buildSources() throws AndrolibException {
if (!buildSourcesRaw("classes.dex") && !buildSourcesSmali("smali", "classes.dex")) {
LOGGER.warning("Could not find sources");
}
}

private void buildNonDefaultSources() throws AndrolibException {
private void scheduleBuildDexFiles() throws AndrolibException {
try {
mWorker.submit(() -> scheduleDexBuild("classes.dex", "smali"));

// loop through any smali_ directories for multi-dex apks
Map<String, Directory> dirs = mApkDir.getDirectory().getDirs();
for (Map.Entry<String, Directory> directory : dirs.entrySet()) {
String name = directory.getKey();
if (name.startsWith("smali_")) {
String filename = name.substring(name.indexOf("_") + 1) + ".dex";

if (!buildSourcesRaw(filename) && !buildSourcesSmali(name, filename)) {
LOGGER.warning("Could not find sources");
}
mWorker.submit(() -> scheduleDexBuild(filename, name));
}
}

Expand All @@ -177,6 +176,19 @@ private void buildNonDefaultSources() throws AndrolibException {
}
}

private void scheduleDexBuild(String filename, String smali) {
try {
if (mBuildError.get() != null) {
return;
}
if (!buildSourcesRaw(filename) && !buildSourcesSmali(smali, filename)) {
LOGGER.warning("Could not find sources");
}
} catch (AndrolibException e) {
mBuildError.compareAndSet(null, e);
}
}

private boolean buildSourcesRaw(String filename) throws AndrolibException {
File working = new File(mApkDir, filename);
if (!working.exists()) {
Expand Down Expand Up @@ -214,6 +226,7 @@ private boolean buildSourcesSmali(String folder, String filename) throws Androli
}

private void buildResources() throws BrutException {
// create res folder, manifest file and resources.arsc
if (!buildResourcesRaw() && !buildResourcesFull() && !buildManifest()) {
LOGGER.warning("Could not find resources");
}
Expand Down Expand Up @@ -375,7 +388,7 @@ private boolean buildManifest() throws BrutException {
}
}

private void buildLibs() throws AndrolibException {
private void copyLibs() throws AndrolibException {
buildLibrary("lib");
buildLibrary("libs");
buildLibrary("kotlin");
Expand All @@ -401,7 +414,7 @@ private void buildLibrary(String folder) throws AndrolibException {
}
}

private void buildCopyOriginalFiles() throws AndrolibException {
private void copyOriginalFilesIfEnabled() throws AndrolibException {
if (mConfig.copyOriginalFiles) {
File originalDir = new File(mApkDir, "original");
if (originalDir.exists()) {
Expand All @@ -427,49 +440,34 @@ private void buildCopyOriginalFiles() throws AndrolibException {
}
}

private void buildUnknownFiles(File outFile) throws AndrolibException {
if (mApkInfo.unknownFiles != null) {
LOGGER.info("Copying unknown files/dir...");

Map<String, String> files = mApkInfo.unknownFiles;
File tempFile = new File(outFile.getParent(), outFile.getName() + ".apktool_temp");
boolean renamed = outFile.renameTo(tempFile);
if (!renamed) {
throw new AndrolibException("Unable to rename temporary file");
}

try (
ZipFile inputFile = new ZipFile(tempFile);
ZipOutputStream actualOutput = new ZipOutputStream(Files.newOutputStream(outFile.toPath()))
) {
copyExistingFiles(inputFile, actualOutput);
copyUnknownFiles(actualOutput, files);
} catch (IOException | BrutException ex) {
throw new AndrolibException(ex);
}

// Remove our temporary file.
private void buildApk(File outApk) throws AndrolibException {
LOGGER.info("Building apk file...");
if (outApk.exists()) {
//noinspection ResultOfMethodCallIgnored
tempFile.delete();
outApk.delete();
} else {
File outDir = outApk.getParentFile();
if (outDir != null && !outDir.exists()) {
//noinspection ResultOfMethodCallIgnored
outDir.mkdirs();
}
}
}

private void copyExistingFiles(ZipFile inputFile, ZipOutputStream outputFile) throws IOException {
// First, copy the contents from the existing outFile:
Enumeration<? extends ZipEntry> entries = inputFile.entries();
while (entries.hasMoreElements()) {
ZipEntry entry = new ZipEntry(entries.nextElement());

// We can't reuse the compressed size because it depends on compression sizes.
entry.setCompressedSize(-1);
outputFile.putNextEntry(entry);
File assetDir = new File(mApkDir, "assets");
if (!assetDir.exists()) {
assetDir = null;
}
try (ZipOutputStream zipOutputStream = new ZipOutputStream(Files.newOutputStream(outApk.toPath()))) {
// zip all AAPT-generated files
ZipUtils.zipFoldersPreserveStream(new File(mApkDir, APK_DIRNAME), zipOutputStream, assetDir, mApkInfo.doNotCompress);

// No need to create directory entries in the final apk
if (!entry.isDirectory()) {
BrutIO.copy(inputFile, outputFile, entry);
// we must copy some files manually
// this is because Aapt won't add files it doesn't know (ex unknown files)
if (mApkInfo.unknownFiles != null) {
LOGGER.info("Copying unknown files/dir...");
copyUnknownFiles(zipOutputStream, mApkInfo.unknownFiles);
}

outputFile.closeEntry();
} catch (IOException | BrutException e) {
throw new AndrolibException(e);
}
}

Expand Down Expand Up @@ -513,33 +511,6 @@ private void copyUnknownFiles(ZipOutputStream outputFile, Map<String, String> fi
}
}

private void buildApk(File outApk) throws AndrolibException {
LOGGER.info("Building apk file...");
if (outApk.exists()) {
//noinspection ResultOfMethodCallIgnored
outApk.delete();
} else {
File outDir = outApk.getParentFile();
if (outDir != null && !outDir.exists()) {
//noinspection ResultOfMethodCallIgnored
outDir.mkdirs();
}
}
File assetDir = new File(mApkDir, "assets");
if (!assetDir.exists()) {
assetDir = null;
}
zipPackage(outApk, new File(mApkDir, APK_DIRNAME), assetDir);
}

private void zipPackage(File apkFile, File rawDir, File assetDir) throws AndrolibException {
try {
ZipUtils.zipFolders(rawDir, apkFile, assetDir, mApkInfo.doNotCompress);
} catch (IOException | BrutException ex) {
throw new AndrolibException(ex);
}
}

private File[] getIncludeFiles() throws AndrolibException {
UsesFramework usesFramework = mApkInfo.usesFramework;
if (usesFramework == null) {
Expand Down
Loading