3.通过调试ue4源码了解ue4打包功能
打包命令的构成
首先要了解到，与pak包相关的操作，使用的命令为unrealpak，该命令最终调用了ue4引擎工程内的unrealpak.exe程序
UnrealPak <PakFilename> -Create=<paklist> [Options]
其中Create表示打包，PakList表示要打包的资源
还有一些其他的常见选项：
Options:-blocksize=<BlockSize>-bitwindow=<BitWindow>-compress-encrypt-order=<OrderingFile>-diff (requires 2 filenames first)-enginedir (specify engine dir for when using ini encryption configs)-projectdir (specify project dir for when using ini encryption configs)-encryptionini (specify ini base name to gather encryption settings from)-encryptindex (encrypt the pak file index, making it unusable in unrealpak without supplying the key)
本次所使用的mypaklist.txt内容
E:\unreal projects\test 4.27\Content\StarterContent\Maps\StarterMap.umapE:\unreal projects\test 4.27\Content\StarterContent\Maps\Minimal_Default.umapE:\unreal projects\test 4.27\Content\StarterContent\Maps\Advanced_Lighting.umap
构造命令：
UnrealPak.exe "E:\unreal projects\test 4.27\pak\mypak.pak" -Create= "E:\unreal projects\test 4.27\mypaklist.txt"
命令执行后会在目标路径生成目标pak文件
功能源码的调试分析
这里通过调试一次打包过程的代码运行，了解了打包流程的实现，细节的地方就写在注释中
先进入的是UnrealPak.cpp，在初始化模块后通过PakFileUtilities.cpp中的ExecuteUnrealPak函数完成打包功能。此后计算执行时间并退出。
INT32_MAIN_INT32_ARGC_TCHAR_ARGV()
{
// 照顺序加载游戏相关的引擎模块、项目和第三方插件等模块
GEngineLoop.PreInit(ArgC, ArgV);
//获取时间戳，记录开始执行的时间点
double StartTime = FPlatformTime::Seconds();
//获取命令行传入的参数并传入关键函数ExecuteUnrealPak，该函数完成pak相关功能
int32 Result = ExecuteUnrealPak(FCommandLine::Get())? 0 : 1;
//打印日志：命令执行所耗时间
UE_LOG(LogPakFile, Display, TEXT("Unreal pak executed in %f seconds"), FPlatformTime::Seconds() - StartTime );
GLog->Flush();
//退出
RequestEngineExit(TEXT("UnrealPak Exiting"));
FEngineLoop::AppPreExit();
FModuleManager::Get().UnloadModulesAtShutdown();
FEngineLoop::AppExit();
return Result;
}
分析源码PakFileUtilities.cpp
PakFileUtilities.cpp中包含了pak相关的功能代码
bool ExecuteUnrealPak(const TCHAR* CmdLine)
//循环解析传入的参数，
TArray<FString> NonOptionArguments;
for (const TCHAR* CmdLineEnd = CmdLine; *CmdLineEnd != 0;)
{
FString Argument = FParse::Token(CmdLineEnd, false);
if (Argument.Len() > 0 && !Argument.StartsWith(TEXT("-")))
{
NonOptionArguments.Add(Argument);
}
}
然后依次是处理option为-Project，-Batch时的情况，注意打包所使用的option为-Create，所以其他option的处理代码不会执行
代码看起来是加解密相关
//一个加解密结构
FKeyChain KeyChain;
LoadKeyChain(CmdLine, KeyChain);
KeyChainUtilities::ApplyEncryptionKeys(KeyChain);
跟踪后看到此处KeyChain属性为空或0，说明没有加解密，那是因为一开始命令就没有指定加密
接下来检查option中是否有-Test和-Vertify
bool IsTestCommand = FParse::Param(CmdLine, TEXT("Test"));
bool IsVerifyCommand = FParse::Param(CmdLine, TEXT("Verify"));
如果有-Test或-Vertify就进行相关处理
if (IsTestCommand || IsVerifyCommand)
{
……
}
后面依次是对-List，Info，Diff，Extract，AuditFiles，WhatsAtOffset，CalcCompressionBlockCRCs，GeneratePIXMappingFile，Repack等等可选的处理
if (FParse::Param(CmdLine, TEXT("List")))
{
……
}
if (FParse::Param(CmdLine, TEXT("Diff")))
{
……
}
if (FParse::Param(CmdLine, TEXT("Extract")))
{
……
}
if (FParse::Param(CmdLine, TEXT("AuditFiles")))
{
……
}
if (FParse::Param(CmdLine, TEXT("WhatsAtOffset")))
{
……
}
if (FParse::Param(CmdLine, TEXT("CalcCompressionBlockCRCs")))
{
……
}
if (FParse::Param(CmdLine, TEXT("GeneratePIXMappingFile")))
{
……
}
if (FParse::Param(CmdLine, TEXT("Repack")))
{
……
}
结束这些可选项的处理代码后，如果有传入参数则执行CheckAndReallocThreadPool，此时传入的参数有两个，一个是pakfilename，一个是paklist
if(NonOptionArguments.Num() > 0)
{
CheckAndReallocThreadPool();
其中CheckAndReallocThreadPool函数获取线程数和最大线程容量，并给线程分配内存空间
void CheckAndReallocThreadPool()
{
if (FPlatformProcess::SupportsMultithreading())
{
//获取线程池中的线程数
const int32 ThreadsSpawned = GThreadPool->GetNumThreads();
//获取机器cpu的核心数
const int32 DesiredThreadCount = NumberOfWorkerThreadsDesired();
//判断是否还有空间再分配给新线程
if (ThreadsSpawned < DesiredThreadCount)
{
UE_LOG(LogPakFile, Log, TEXT("Engine only spawned %d worker threads, bumping up to %d!"), ThreadsSpawned, DesiredThreadCount);
GThreadPool->Destroy();
GThreadPool = FQueuedThreadPool::Allocate();
verify(GThreadPool->Create(DesiredThreadCount, 128 * 1024));
}
else
{
UE_LOG(LogPakFile, Log, TEXT("Continuing with %d spawned worker threads."), ThreadsSpawned);
}
}
}
后面进行pak前的准备
//获取pak文件指定的文件名
FString PakFilename = GetPakPath(*NonOptionArguments[0], true);
//生成FPakInputPair结构，存储被打包文件的信息
TArray<FPakInputPair> Entries;
//创建命令行结构
FPakCommandLineParameters CmdLineParameters;
//列出所有要打包进pak文件的文件，解析传入参数，并依次设置上文命令行的属性
ProcessCommandLine(CmdLine, NonOptionArguments, Entries, CmdLineParameters);
ProcessCommandLine函数中，先设置了传入的命令行对象的属性
if (FParse::Value(CmdLine, TEXT("-compressionblocksize="), CompBlockSizeString) &&
FParse::Value(CmdLine, TEXT("-compressionblocksize="), CmdLineParameters.CompressionBlockSize))
{
……
}
if (!FParse::Value(CmdLine, TEXT("-patchpaddingalign="), CmdLineParameters.PatchFilePadAlign))
{
……
}
if (!FParse::Value(CmdLine, TEXT("-AlignForMemoryMapping="), CmdLineParameters.AlignForMemoryMapping))
{
……
}
if (FParse::Param(CmdLine, TEXT("encryptindex")))
{
……
}
if (FParse::Param(CmdLine, TEXT("sign")))
{
……
}
if (FParse::Param(CmdLine, TEXT("AlignFilesLargerThanBlock")))
{
……
}
if (FParse::Param(CmdLine, TEXT("ForceCompress")))
{
……
}
if (FParse::Param(CmdLine, TEXT("FileRegions")))
{
……
}
如果传入参数中有-compressionformats 或-compressionformat，就 默认使用 ZLib 压缩算法压缩
if (FParse::Value(CmdLine, TEXT("-compressionformats="), DesiredCompressionFormats) || FParse::Value(CmdLine, TEXT("-compressionformat="), DesiredCompressionFormats))
{
TArray<FString> Formats;
DesiredCompressionFormats.ParseIntoArray(Formats, TEXT(","));
for (FString& Format : Formats)
{
// look until we have a valid format
FName FormatName = *Format;
if (FCompression::IsFormatValid(FormatName))
{
CmdLineParameters.CompressionFormats.Add(FormatName);
break;
}
else
{
UE_LOG(LogPakFile, Warning, TEXT("Compression format %s is not recognized"), *Format);
}
}
}
判断是否要压缩和加密
if (FParse::Value(CmdLine, TEXT("-create="), ResponseFile))
{
……
bool bCompress = FParse::Param(CmdLine, TEXT("compress"));
if ( bCompress )
……
bool bEncrypt = FParse::Param(CmdLine, TEXT("encrypt"));
if (CmdLineParameters.GeneratePatch)
……
结束ProcessCommandLine函数后回到主程序，按参数依次判断执行刚刚的命令
if (FParse::Value(CmdLine, TEXT("-order="), GameOpenOrderStr, false))
{
……}
if (FParse::Value(CmdLine, TEXT("-secondaryOrder="), SecondOrderStr, false))
{
……
}
if ( CmdLineParameters.GeneratePatch )
{
if (!FParse::Value(CmdLine, TEXT("TempFiles="), OutputPath))
{
……
}
if (FParse::Value(FCommandLine::Get(), TEXT("PatchCryptoKeys="), PatchReferenceCryptoKeysFilename))
{
……
}
……
}
接下来创建一个空数组FilesToAdd，并通过函数CollectFilesToAdd开始收集需要打包的文件。先打印日志和记录开始时间
TArray<FPakInputPair> FilesToAdd;
CollectFilesToAdd(FilesToAdd, Entries, OrderMap, CmdLineParameters);
UE_LOG(LogPakFile, Display, TEXT("Collecting files to add to pak file..."));
const double StartTime = FPlatformTime::Seconds();
用for循环遍历FPakInputPair内容，也就是每个文件的信息
for (int32 Index = 0; Index < InEntries.Num(); Index++)
从input中解析参数，确定是否压缩和加密。
const FPakInputPair& Input = InEntries[Index];
const FString& Source = Input.Source;
bool bCompression = Input.bNeedsCompression;
bool bEncryption = Input.bNeedEncryption;
获取文件名和文件路径，并进行检查和格式化处理
FString Filename = FPaths::GetCleanFilename(Source);
FString Directory = FPaths::GetPath(Source);
FPaths::MakeStandardFilename(Directory);
FPakFile::MakeDirectoryFromPath(Directory);
因为paklist文件中可以使用通配符，所以接着是对通配符的相应解析代码
if (Filename.IsEmpty())
{
Filename = TEXT("*.*");
}
if ( Filename.Contains(TEXT("*")) )
{
……//通配符*匹配多个文件
}
else
{
……//不包含*则匹配特定单个文件
}
其中对多个文件的处理，分为格式化文件名，文件排序，压缩，加密等
// Add multiple files
TArray<FString> FoundFiles;
IFileManager::Get().FindFilesRecursive(FoundFiles, *Directory, *Filename, true, false);
for (int32 FileIndex = 0; FileIndex < FoundFiles.Num(); FileIndex++)
{
//格式化文件名和路径
FPakInputPair FileInput;
FileInput.Source = FoundFiles[FileIndex];
FPaths::MakeStandardFilename(FileInput.Source);
FileInput.Dest = FileInput.Source.Replace(*Directory, *Input.Dest, ESearchCase::IgnoreCase);
//对文件进行排序
uint64 FileOrder = OrderMap.GetFileOrder(FileInput.Dest, false, &FileInput.bIsInPrimaryOrder);
if(FileOrder != MAX_uint64)
{
FileInput.SuggestedOrder = FileOrder;
}
else
{
// we will put all unordered files at 1 << 28 so that they are before any uexp or ubulk files we assign orders to here
FileInput.SuggestedOrder = (1 << 28);
// if this is a cook order or an old order it will not have uexp files in it, so we put those in the same relative order after all of the normal files, but before any ubulk files
if (FileInput.Dest.EndsWith(TEXT("uexp")) || FileInput.Dest.EndsWith(TEXT("ubulk")))
{
FileOrder = OrderMap.GetFileOrder(FPaths::GetBaseFilename(FileInput.Dest, false) + TEXT(".uasset"), false, &FileInput.bIsInPrimaryOrder);
if (FileOrder == MAX_uint64)
{
FileOrder = OrderMap.GetFileOrder(FPaths::GetBaseFilename(FileInput.Dest, false) + TEXT(".umap"), false, &FileInput.bIsInPrimaryOrder);
}
if (FileInput.Dest.EndsWith(TEXT("uexp")))
{
FileInput.SuggestedOrder = ((FileOrder != MAX_uint64) ? FileOrder : 0) + (1 << 29);
}
else
{
FileInput.SuggestedOrder = ((FileOrder != MAX_uint64) ? FileOrder : 0) + (1 << 30);
}
}
}
//是否压缩和加密
FileInput.bNeedsCompression = bCompression;
FileInput.bNeedEncryption = bEncryption;
if (!AddedFiles.Contains(FileInput.Source))
{
OutFilesToAdd.Add(FileInput);
AddedFiles.Add(FileInput.Source);
}
else
{
int32 FoundIndex;
OutFilesToAdd.Find(FileInput,FoundIndex);
OutFilesToAdd[FoundIndex].bNeedEncryption |= bEncryption;
OutFilesToAdd[FoundIndex].bNeedsCompression |= bCompression;
OutFilesToAdd[FoundIndex].SuggestedOrder = FMath::Min<uint64>(OutFilesToAdd[FoundIndex].SuggestedOrder, FileInput.SuggestedOrder);
}
}
如果paklist中使用的不是*这样的通配符，而是具体的文件名，就对单独文件进行文件名格式化，排序，压缩，加密的处理
// Add single file
FPakInputPair FileInput;
FileInput.Source = Input.Source;
FPaths::MakeStandardFilename(FileInput.Source);
FileInput.Dest = FileInput.Source.Replace(*Directory, *Input.Dest, ESearchCase::IgnoreCase);
uint64 FileOrder = OrderMap.GetFileOrder(FileInput.Dest, CmdLineParameters.bFallbackOrderForNonUassetFiles, &FileInput.bIsInPrimaryOrder);
if (FileOrder != MAX_uint64)
{
FileInput.SuggestedOrder = FileOrder;
}
FileInput.bNeedEncryption = bEncryption;
FileInput.bNeedsCompression = bCompression;
if (AddedFiles.Contains(FileInput.Source))
{
int32 FoundIndex;
OutFilesToAdd.Find(FileInput, FoundIndex);
OutFilesToAdd[FoundIndex].bNeedEncryption |= bEncryption;
OutFilesToAdd[FoundIndex].bNeedsCompression |= bCompression;
OutFilesToAdd[FoundIndex].SuggestedOrder = FMath::Min<uint64>(OutFilesToAdd[FoundIndex].SuggestedOrder, FileInput.SuggestedOrder);
}
else
{
OutFilesToAdd.Add(FileInput);
AddedFiles.Add(FileInput.Source);
}
最后这些文件放在字符串集合AddedFiles中，按要求的顺序排序，然后按字母顺序排序
struct FInputPairSort
{
FORCEINLINE bool operator()(const FPakInputPair& A, const FPakInputPair& B) const
{
return A.bIsDeleteRecord == B.bIsDeleteRecord ? (A.SuggestedOrder == B.SuggestedOrder ? A.Dest < B.Dest : A.SuggestedOrder < B.SuggestedOrder) : A.bIsDeleteRecord < B.bIsDeleteRecord;
}
};
OutFilesToAdd.Sort(FInputPairSort());
再在日志中输出用时
UE_LOG(LogPakFile, Display, TEXT("Collected %d files in %.2lfs."), OutFilesToAdd.Num(), FPlatformTime::Seconds() - StartTime);
到这里就结束了CollectFilesToAdd函数，返回后调用CreatePakFile创建pak文件，最后返回create的成败并退出
bool bResult = CreatePakFile(*PakFilename, FilesToAdd, CmdLineParameters, KeyChain); if (CmdLineParameters.GeneratePatch) { …… } GetDerivedDataCacheRef().WaitForQuiescence(true);
这时控制台打印出了日志，显示已成功读取list文件，并获取三个预备打包的文件
继续来看CreatePakFile函数，先实现了传入文件属性的解析，包含文件路径，是否压缩，是否加密等l
//创建字符串集合来存储要打包的文件名 TSet<FString> AddedFiles; //遍历传入的OutFilesToAdd字符串集合，也就是读取list的结果 for (int32 Index = 0; Index < InEntries.Num(); Index++) { //解析出文件路径，是否压缩，是否加密等属性 const FPakInputPair& Input = InEntries[Index]; const FString& Source = Input.Source; bool bCompression = Input.bNeedsCompression; bool bEncryption = Input.bNeedEncryption;
再通过文件的bIsDeleteRecord属性判断该文件在新的补丁处是否要被删除，补丁可以理解为对旧的pak的更新
if (Input.bIsDeleteRecord)
{
OutFilesToAdd.Add(Input);
continue;
}
获取文件的文件名，路径。并将其格式化，判断是否为空
FString Filename = FPaths::GetCleanFilename(Source);
FString Directory = FPaths::GetPath(Source);
FPaths::MakeStandardFilename(Directory);
FPakFile::MakeDirectoryFromPath(Directory);
if (Filename.IsEmpty())
{
Filename = TEXT("*.*");
}
这时同样对文件名做两种情况处理，通配符情况，和具体文件名情况。
先处理通配符描述的情况，主要进行了文件属性解析，排序（按照指定排序，未指定则按照字母表顺序排序。），判断文件是否重复。
if ( Filename.Contains(TEXT("*")) )
{
TArray<FString> FoundFiles;
IFileManager::Get().FindFilesRecursive(FoundFiles, *Directory, *Filename, true, false);
for (int32 FileIndex = 0; FileIndex < FoundFiles.Num(); FileIndex++)
{
//解析文件各项属性
FPakInputPair FileInput;
FileInput.Source = FoundFiles[FileIndex];
FPaths::MakeStandardFilename(FileInput.Source);
FileInput.Dest = FileInput.Source.Replace(*Directory, *Input.Dest, ESearchCase::IgnoreCase);
//定义文件的排序序号
uint64 FileOrder = OrderMap.GetFileOrder(FileInput.Dest, false, &FileInput.bIsInPrimaryOrder);
//如果有指定序号，按照指定的序号排序；否则默认MAX_uint64，再按照字母表顺序排序。
if(FileOrder != MAX_uint64)
{
FileInput.SuggestedOrder = FileOrder;
}
else
{
// we will put all unordered files at 1 << 28 so that they are before any uexp or ubulk files we assign orders to here
FileInput.SuggestedOrder = (1 << 28);
// if this is a cook order or an old order it will not have uexp files in it, so we put those in the same relative order after all of the normal files, but before any ubulk files
if (FileInput.Dest.EndsWith(TEXT("uexp")) || FileInput.Dest.EndsWith(TEXT("ubulk")))
{
FileOrder = OrderMap.GetFileOrder(FPaths::GetBaseFilename(FileInput.Dest, false) + TEXT(".uasset"), false, &FileInput.bIsInPrimaryOrder);
if (FileOrder == MAX_uint64)
{
FileOrder = OrderMap.GetFileOrder(FPaths::GetBaseFilename(FileInput.Dest, false) + TEXT(".umap"), false, &FileInput.bIsInPrimaryOrder);
}
if (FileInput.Dest.EndsWith(TEXT("uexp")))
{
FileInput.SuggestedOrder = ((FileOrder != MAX_uint64) ? FileOrder : 0) + (1 << 29);
}
else
{
FileInput.SuggestedOrder = ((FileOrder != MAX_uint64) ? FileOrder : 0) + (1 << 30);
}
}
}
//解析加密、压缩属性。判断文件是否重复，未重复就直接加入文件集合，重复就进行属性比对。
FileInput.bNeedsCompression = bCompression;
FileInput.bNeedEncryption = bEncryption;
if (!AddedFiles.Contains(FileInput.Source))
{
OutFilesToAdd.Add(FileInput);
AddedFiles.Add(FileInput.Source);
}
else
{
int32 FoundIndex;
OutFilesToAdd.Find(FileInput,FoundIndex);
OutFilesToAdd[FoundIndex].bNeedEncryption |= bEncryption;
OutFilesToAdd[FoundIndex].bNeedsCompression |= bCompression;
OutFilesToAdd[FoundIndex].SuggestedOrder = FMath::Min<uint64>(OutFilesToAdd[FoundIndex].SuggestedOrder, FileInput.SuggestedOrder);
}
}
}
同上，完成特定文件情况下的排序处理，属性解析，重复文件的属性确认
else
{
//获取预备打包文件的源地址和目的地址，以及排序方式
FPakInputPair FileInput;
FileInput.Source = Input.Source;
FPaths::MakeStandardFilename(FileInput.Source);
FileInput.Dest = FileInput.Source.Replace(*Directory, *Input.Dest, ESearchCase::IgnoreCase);
uint64 FileOrder = OrderMap.GetFileOrder(FileInput.Dest, CmdLineParameters.bFallbackOrderForNonUassetFiles, &FileInput.bIsInPrimaryOrder);
//指定了排序序号的情况
if (FileOrder != MAX_uint64)
{
FileInput.SuggestedOrder = FileOrder;
}
//压缩、加密属性的解析
FileInput.bNeedEncryption = bEncryption;
FileInput.bNeedsCompression = bCompression;
//如果要加入的文件已经存在于集合中，就不再重复加入，但是要比对属性无误
if (AddedFiles.Contains(FileInput.Source))
{
int32 FoundIndex;
OutFilesToAdd.Find(FileInput, FoundIndex);
OutFilesToAdd[FoundIndex].bNeedEncryption |= bEncryption;
OutFilesToAdd[FoundIndex].bNeedsCompression |= bCompression;
OutFilesToAdd[FoundIndex].SuggestedOrder = FMath::Min<uint64>(OutFilesToAdd[FoundIndex].SuggestedOrder, FileInput.SuggestedOrder);
}
//不重复就直接加入
else
{
OutFilesToAdd.Add(FileInput);
AddedFiles.Add(FileInput.Source);
}
}
最后进行排序，结束pak文件的生成
// Sort by suggested order then alphabetically
struct FInputPairSort
{
FORCEINLINE bool operator()(const FPakInputPair& A, const FPakInputPair& B) const
{
return A.bIsDeleteRecord == B.bIsDeleteRecord ? (A.SuggestedOrder == B.SuggestedOrder ? A.Dest < B.Dest : A.SuggestedOrder < B.SuggestedOrder) : A.bIsDeleteRecord < B.bIsDeleteRecord;
}
};
OutFilesToAdd.Sort(FInputPairSort());
UE_LOG(LogPakFile, Display, TEXT("Collected %d files in %.2lfs."), OutFilesToAdd.Num(), FPlatformTime::Seconds() - StartTime);
收尾工作，通过文件名的路径找到临时文件夹并删除
if (CmdLineParameters.GeneratePatch)
{
FString OutputPath = FPaths::GetPath(PakFilename) / FString(TEXT("TempFiles"));
IFileManager::Get().DeleteDirectory(*OutputPath, false, true);
}
GetDerivedDataCacheRef().WaitForQuiescence(true);
到这里打包功能代码执行完毕，可以看到控制台打印出日志，从数据角度描述了刚刚的执行过程
最后计算用时并退出
UE_LOG(LogPakFile, Display, TEXT("Unreal pak executed in %f seconds"), FPlatformTime::Seconds() - StartTime ); GLog->Flush(); RequestEngineExit(TEXT("UnrealPak Exiting")); FEngineLoop::AppPreExit(); FModuleManager::Get().UnloadModulesAtShutdown(); FEngineLoop::AppExit();
