fastbuild进行ue4 shader连编

参考https://github.com/fastbuild/fastbuild/issues/539中

wsj0000 上传的代码，

根据

wout276所说，将

FShaderCompileUtilities::DoWriteTasks in ShaderCompiler.cpp里的:

for (TPair<FString, FString>& Pair : ShaderSourceDirectoryMappings)
{
    Pair.Value = FPaths::ConvertRelativePathToFull(Pair.Value);
}

注释掉。

https://blog.kangkang.org/index.php/archives/409这篇文章说了编译机器上要安装一些东西，笔者因为是在别人基础上改的，不知道是否要安装，可能也是要安装的。这篇文章中其他的东西好像也没啥卵用，不按照它那个搞也没啥问题。

我用的是ue4.22版本，下面是代码文件：

shaderCompile.h文件，增加了FShaderCompileThreadRunnableBase类和文件相关函数，FShaderCompilingManager里增加FShaderCompileFASTBuildThreadRunnable，其他不变：

 

 

 

 

 

 

// Copyright 1998-2019 Epic Games, Inc. All Rights Reserved.

/*=============================================================================
    ShaderCompiler.h: Platform independent shader compilation definitions.
=============================================================================*/

#pragma once

#include "CoreMinimal.h"
#include "Templates/RefCounting.h"
#include "Templates/ScopedPointer.h"
#include "HAL/PlatformProcess.h"
#include "ShaderCore.h"
#include "Shader.h"
#include "HAL/RunnableThread.h"
#include "HAL/Runnable.h"
#include "Templates/Atomic.h"
#include "Templates/UniquePtr.h"

class FShaderCompileJob;
class FShaderPipelineCompileJob;
class FVertexFactoryType;

DECLARE_LOG_CATEGORY_EXTERN(LogShaderCompilers, Log, All);

class FShaderCompileJob;
class FShaderPipelineCompileJob;

#define DEBUG_INFINITESHADERCOMPILE 0


/** Stores all of the common information used to compile a shader or pipeline. */
class FShaderCommonCompileJob : public FRefCountedObject
{
public:
    /** Id of the shader map this shader belongs to. */
    uint32 Id;
    /** true if the results of the shader compile have been processed. */
    bool bFinalized;
    /** Output of the shader compile */
    bool bSucceeded;
    bool bOptimizeForLowLatency;

    FShaderCommonCompileJob(uint32 InId) :
        Id(InId),
        bFinalized(false),
        bSucceeded(false),
        bOptimizeForLowLatency(false)
    {
    }

    virtual ~FShaderCommonCompileJob() {}

    virtual FShaderCompileJob* GetSingleShaderJob() { return nullptr; }
    virtual const FShaderCompileJob* GetSingleShaderJob() const { return nullptr; }
    virtual FShaderPipelineCompileJob* GetShaderPipelineJob() { return nullptr; }
    virtual const FShaderPipelineCompileJob* GetShaderPipelineJob() const { return nullptr; }
};


/** Stores all of the input and output information used to compile a single shader. */
class FShaderCompileJob : public FShaderCommonCompileJob
{
public:
    /** Vertex factory type that this shader belongs to, may be NULL */
    FVertexFactoryType* VFType;
    /** Shader type that this shader belongs to, must be valid */
    FShaderType* ShaderType;
    /** Unique permutation identifier of the global shader type. */
    int32 PermutationId;
    /** Input for the shader compile */
    FShaderCompilerInput Input;
    FShaderCompilerOutput Output;

    // List of pipelines that are sharing this job.
    TMap<const FVertexFactoryType*, TArray<const FShaderPipelineType*>> SharingPipelines;

    FShaderCompileJob(uint32 InId, FVertexFactoryType* InVFType, FShaderType* InShaderType, int32 InPermutationId) :
        FShaderCommonCompileJob(InId),
        VFType(InVFType),
        ShaderType(InShaderType),
        PermutationId(InPermutationId)
    {
    }

    virtual FShaderCompileJob* GetSingleShaderJob() override { return this; }
    virtual const FShaderCompileJob* GetSingleShaderJob() const override { return this; }
};

class FShaderPipelineCompileJob : public FShaderCommonCompileJob
{
public:
    TArray<FShaderCommonCompileJob*> StageJobs;
    bool bFailedRemovingUnused;

    /** Shader pipeline that this shader belongs to, may (currently) be NULL */
    const FShaderPipelineType* ShaderPipeline;

    FShaderPipelineCompileJob(uint32 InId, const FShaderPipelineType* InShaderPipeline, int32 NumStages) :
        FShaderCommonCompileJob(InId),
        bFailedRemovingUnused(false),
        ShaderPipeline(InShaderPipeline)
    {
        check(InShaderPipeline && InShaderPipeline->GetName());
        check(NumStages > 0);
        StageJobs.Empty(NumStages);
    }

    ~FShaderPipelineCompileJob()
    {
        for (int32 Index = 0; Index < StageJobs.Num(); ++Index)
        {
            delete StageJobs[Index];
        }
        StageJobs.Reset();
    }

    virtual FShaderPipelineCompileJob* GetShaderPipelineJob() override { return this; }
    virtual const FShaderPipelineCompileJob* GetShaderPipelineJob() const override { return this; }
};

class FGlobalShaderTypeCompiler
{
public:
    /**
    * Enqueues compilation of a shader of this type.
    */
    ENGINE_API static class FShaderCompileJob* BeginCompileShader(FGlobalShaderType* ShaderType, int32 PermutationId, EShaderPlatform Platform, const FShaderPipelineType* ShaderPipeline, TArray<FShaderCommonCompileJob*>& NewJobs);

    /**
    * Enqueues compilation of a shader pipeline of this type.
    */
    ENGINE_API static void BeginCompileShaderPipeline(EShaderPlatform Platform, const FShaderPipelineType* ShaderPipeline, const TArray<FGlobalShaderType*>& ShaderStages, TArray<FShaderCommonCompileJob*>& NewJobs);

    /** Either returns an equivalent existing shader of this type, or constructs a new instance. */
    static FShader* FinishCompileShader(FGlobalShaderType* ShaderType, const FShaderCompileJob& CompileJob, const FShaderPipelineType* ShaderPipelineType);
};

class FShaderCompileThreadRunnableBase : public FRunnable
{
    friend class FShaderCompilingManager;

protected:
    /** The manager for this thread */
    class FShaderCompilingManager* Manager;
    /** The runnable thread */
    FRunnableThread* Thread;
    
    /** If the thread has been terminated by an unhandled exception, this contains the error message. */
    FString ErrorMessage;
    /** true if the thread has been terminated by an unhandled exception. */
    bool bTerminatedByError;

    TAtomic<bool> bForceFinish;

public:
    FShaderCompileThreadRunnableBase(class FShaderCompilingManager* InManager);
    virtual ~FShaderCompileThreadRunnableBase()
    {}

    void StartThread();

    // FRunnable interface.
    virtual void Stop() { bForceFinish = true; }
    virtual uint32 Run();
    inline void WaitForCompletion() const
    {
        if( Thread )
        {
            Thread->WaitForCompletion();
        }
    }

    /** Checks the thread's health, and passes on any errors that have occured.  Called by the main thread. */
    void CheckHealth() const;

    /** Main work loop. */
    virtual int32 CompilingLoop() = 0;
};

/** 
 * Shader compiling thread
 * This runs in the background while UE4 is running, launches shader compile worker processes when necessary, and feeds them inputs and reads back the outputs.
 */
class FShaderCompileThreadRunnable : public FShaderCompileThreadRunnableBase
{
    friend class FShaderCompilingManager;
private:

    /** Information about the active workers that this thread is tracking. */
    TArray<struct FShaderCompileWorkerInfo*> WorkerInfos;
    /** Tracks the last time that this thread checked if the workers were still active. */
    double LastCheckForWorkersTime;

public:

    /** Initialization constructor. */
    FShaderCompileThreadRunnable(class FShaderCompilingManager* InManager);
    virtual ~FShaderCompileThreadRunnable();

private:

    /** 
     * Grabs tasks from Manager->CompileQueue in a thread safe way and puts them into QueuedJobs of available workers. 
     * Also writes completed jobs to Manager->ShaderMapJobs.
     */
    int32 PullTasksFromQueue();

    /** Used when compiling through workers, writes out the worker inputs for any new tasks in WorkerInfos.QueuedJobs. */
    void WriteNewTasks();

    /** Used when compiling through workers, launches worker processes if needed. */
    bool LaunchWorkersIfNeeded();

    /** Used when compiling through workers, attempts to open the worker output file if the worker is done and read the results. */
    void ReadAvailableResults();

    /** Used when compiling directly through the console tools dll. */
    void CompileDirectlyThroughDll();

    /** Main work loop. */
    virtual int32 CompilingLoop() override;
};


namespace FShaderCompileUtilities
{
    bool DoWriteTasks(const TArray<FShaderCommonCompileJob*>& QueuedJobs, FArchive& TransferFile);
    void DoReadTaskResults(const TArray<FShaderCommonCompileJob*>& QueuedJobs, FArchive& OutputFile);
    FArchive* CreateFileHelper(const FString& Filename);
    void MoveFileHelper(const FString& To, const FString& From);
    void DeleteFileHelper(const FString& Filename);
}


#if PLATFORM_WINDOWS // FASTBuild shader compilation is only supported on Windows.
class FShaderCompileFASTBuildThreadRunnable : public FShaderCompileThreadRunnableBase
{
private:
    /** The handle referring to the FASTBuild console process, if a build is in progress. */
    FProcHandle BuildProcessHandle;

    /** Process ID of the FASTBuild console, if a build is in progress. */
    uint32 BuildProcessID;

    /**
    * A map of directory paths to shader jobs contained within that directory.
    * One entry per FASTBuild task.
    */
    class FShaderBatch
    {
        TArray<FShaderCommonCompileJob*> Jobs;
        bool bTransferFileWritten;

    public:
        bool bSuccessfullyCompleted;
        const FString& DirectoryBase;
        const FString& InputFileName;
        const FString& SuccessFileName;
        const FString& OutputFileName;

        int32 BatchIndex;
        int32 DirectoryIndex;

        FString WorkingDirectory;
        FString OutputFileNameAndPath;
        FString SuccessFileNameAndPath;
        FString InputFileNameAndPath;

        FShaderBatch(const FString& InDirectoryBase, const FString& InInputFileName, const FString& InSuccessFileName, const FString& InOutputFileName, int32 InDirectoryIndex, int32 InBatchIndex)
            : bTransferFileWritten(false)
            , bSuccessfullyCompleted(false)
            , DirectoryBase(InDirectoryBase)
            , InputFileName(InInputFileName)
            , SuccessFileName(InSuccessFileName)
            , OutputFileName(InOutputFileName)
        {
            SetIndices(InDirectoryIndex, InBatchIndex);
        }

        void SetIndices(int32 InDirectoryIndex, int32 InBatchIndex);

        void CleanUpFiles(bool keepInputFile);

        inline int32 NumJobs()
        {
            return Jobs.Num();
        }
        inline const TArray<FShaderCommonCompileJob*>& GetJobs() const
        {
            return Jobs;
        }

        void AddJob(FShaderCommonCompileJob* Job);

        void WriteTransferFile();
    };
    TArray<FShaderBatch*> ShaderBatchesInFlight;
    int32 ShaderBatchesInFlightCompleted;
    TArray<FShaderBatch*> ShaderBatchesFull;
    TSparseArray<FShaderBatch*> ShaderBatchesIncomplete;

    /** The full path to the two working directories for FASTBuild shader builds. */
    const FString FASTBuildWorkingDirectory;
    uint32 FASTBuildDirectoryIndex;

    uint64 LastAddTime;
    uint64 StartTime;
    int32 BatchIndexToCreate;
    int32 BatchIndexToFill;

    FDateTime ScriptFileCreationTime;

    void PostCompletedJobsForBatch(FShaderBatch* Batch);

    void GatherResultsFromFASTBuild();

public:
    /** Initialization constructor. */
    FShaderCompileFASTBuildThreadRunnable(class FShaderCompilingManager* InManager);
    virtual ~FShaderCompileFASTBuildThreadRunnable();

    /** Main work loop. */
    virtual int32 CompilingLoop() override;

    static bool IsSupported();
};
#endif

#if PLATFORM_WINDOWS // XGE shader compilation is only supported on Windows.

class FShaderCompileXGEThreadRunnable_XmlInterface : public FShaderCompileThreadRunnableBase
{
private:
    /** The handle referring to the XGE console process, if a build is in progress. */
    FProcHandle BuildProcessHandle;
    
    /** Process ID of the XGE console, if a build is in progress. */
    uint32 BuildProcessID;

    /**
     * A map of directory paths to shader jobs contained within that directory.
     * One entry per XGE task.
     */
    class FShaderBatch
    {
        TArray<FShaderCommonCompileJob*> Jobs;
        bool bTransferFileWritten;

    public:
        const FString& DirectoryBase;
        const FString& InputFileName;
        const FString& SuccessFileName;
        const FString& OutputFileName;

        int32 BatchIndex;
        int32 DirectoryIndex;

        FString WorkingDirectory;
        FString OutputFileNameAndPath;
        FString SuccessFileNameAndPath;
        FString InputFileNameAndPath;
        
        FShaderBatch(const FString& InDirectoryBase, const FString& InInputFileName, const FString& InSuccessFileName, const FString& InOutputFileName, int32 InDirectoryIndex, int32 InBatchIndex)
            : bTransferFileWritten(false)
            , DirectoryBase(InDirectoryBase)
            , InputFileName(InInputFileName)
            , SuccessFileName(InSuccessFileName)
            , OutputFileName(InOutputFileName)
        {
            SetIndices(InDirectoryIndex, InBatchIndex);
        }

        void SetIndices(int32 InDirectoryIndex, int32 InBatchIndex);

        void CleanUpFiles(bool keepInputFile);

        inline int32 NumJobs()
        {
            return Jobs.Num();
        }
        inline const TArray<FShaderCommonCompileJob*>& GetJobs() const
        {
            return Jobs;
        }

        void AddJob(FShaderCommonCompileJob* Job);
        
        void WriteTransferFile();
    };
    TArray<FShaderBatch*> ShaderBatchesInFlight;
    TArray<FShaderBatch*> ShaderBatchesFull;
    TSparseArray<FShaderBatch*> ShaderBatchesIncomplete;

    /** The full path to the two working directories for XGE shader builds. */
    const FString XGEWorkingDirectory;
    uint32 XGEDirectoryIndex;

    uint64 LastAddTime;
    uint64 StartTime;
    int32 BatchIndexToCreate;
    int32 BatchIndexToFill;

    FDateTime ScriptFileCreationTime;

    void PostCompletedJobsForBatch(FShaderBatch* Batch);

    void GatherResultsFromXGE();

public:
    /** Initialization constructor. */
    FShaderCompileXGEThreadRunnable_XmlInterface(class FShaderCompilingManager* InManager);
    virtual ~FShaderCompileXGEThreadRunnable_XmlInterface();

    /** Main work loop. */
    virtual int32 CompilingLoop() override;

    static bool IsSupported();
};

class FShaderCompileXGEThreadRunnable_InterceptionInterface : public FShaderCompileThreadRunnableBase
{
    uint32 NumDispatchedJobs;

    TSparseArray<class FXGEShaderCompilerTask*> DispatchedTasks;

public:
    /** Initialization constructor. */
    FShaderCompileXGEThreadRunnable_InterceptionInterface(class FShaderCompilingManager* InManager);
    virtual ~FShaderCompileXGEThreadRunnable_InterceptionInterface();

    /** Main work loop. */
    virtual int32 CompilingLoop() override;

    static bool IsSupported();

private:
    void DispatchShaderCompileJobsBatch(TArray<FShaderCommonCompileJob*>& JobsToSerialize);
};

#endif // PLATFORM_WINDOWS

/** Results for a single compiled shader map. */
struct FShaderMapCompileResults
{
    FShaderMapCompileResults() :
        NumJobsQueued(0),
        bAllJobsSucceeded(true),
        bApplyCompletedShaderMapForRendering(true),
        bRecreateComponentRenderStateOnCompletion(false)
    {}

    int32 NumJobsQueued;
    bool bAllJobsSucceeded;
    bool bApplyCompletedShaderMapForRendering;
    bool bRecreateComponentRenderStateOnCompletion;
    TArray<FShaderCommonCompileJob*> FinishedJobs;
};

/** Results for a single compiled and finalized shader map. */
struct FShaderMapFinalizeResults : public FShaderMapCompileResults
{
    /** Tracks finalization progress on this shader map. */
    int32 FinalizeJobIndex;

    // List of pipelines with shared shaders; nullptr for non mesh pipelines
    TMap<const FVertexFactoryType*, TArray<const FShaderPipelineType*> > SharedPipelines;

    FShaderMapFinalizeResults(const FShaderMapCompileResults& InCompileResults) :
        FShaderMapCompileResults(InCompileResults),
        FinalizeJobIndex(0)
    {}
};

/**  
 * Manager of asynchronous and parallel shader compilation.
 * This class contains an interface to enqueue and retreive asynchronous shader jobs, and manages a FShaderCompileThreadRunnable.
 */
class FShaderCompilingManager
{
    friend class FShaderCompileThreadRunnableBase;
    friend class FShaderCompileThreadRunnable;

#if PLATFORM_WINDOWS
    friend class FShaderCompileXGEThreadRunnable_XmlInterface;
    friend class FShaderCompileXGEThreadRunnable_InterceptionInterface;
    friend class FShaderCompileFASTBuildThreadRunnable;
#endif // PLATFORM_WINDOWS

private:

    //////////////////////////////////////////////////////
    // Thread shared properties: These variables can only be read from or written to when a lock on CompileQueueSection is obtained, since they are used by both threads.

    /** Tracks whether we are compiling while the game is running.  If true, we need to throttle down shader compiling CPU usage to avoid starving the runtime threads. */
    bool bCompilingDuringGame;
    /** Queue of tasks that haven't been assigned to a worker yet. */
    TArray<FShaderCommonCompileJob*> CompileQueue;
    /** Map from shader map Id to the compile results for that map, used to gather compiled results. */
    TMap<int32, FShaderMapCompileResults> ShaderMapJobs;

    /** Number of jobs currently being compiled.  This includes CompileQueue and any jobs that have been assigned to workers but aren't complete yet. */
    int32 NumOutstandingJobs;

    /** Number of jobs currently being compiled.  This includes CompileQueue and any jobs that have been assigned to workers but aren't complete yet. */
    int32 NumExternalJobs;

    /** Critical section used to gain access to the variables above that are shared by both the main thread and the FShaderCompileThreadRunnable. */
    FCriticalSection CompileQueueSection;

    //////////////////////////////////////////////////////
    // Main thread state - These are only accessed on the main thread and used to track progress

    /** Map from shader map id to results being finalized.  Used to track shader finalizations over multiple frames. */
    TMap<int32, FShaderMapFinalizeResults> PendingFinalizeShaderMaps;

    /** The threads spawned for shader compiling. */
    TUniquePtr<FShaderCompileThreadRunnableBase> Thread;

    //////////////////////////////////////////////////////
    // Configuration properties - these are set only on initialization and can be read from either thread

    /** Number of busy threads to use for shader compiling while loading. */
    uint32 NumShaderCompilingThreads;
    /** Number of busy threads to use for shader compiling while in game. */
    uint32 NumShaderCompilingThreadsDuringGame;
    /** Largest number of jobs that can be put in the same batch. */
    int32 MaxShaderJobBatchSize;
    /** Process Id of UE4. */
    uint32 ProcessId;
    /** Whether to allow compiling shaders through the worker application, which allows multiple cores to be used. */
    bool bAllowCompilingThroughWorkers;
    /** Whether to allow shaders to compile in the background or to block after each material. */
    bool bAllowAsynchronousShaderCompiling;
    /** Whether to ask to retry a failed shader compile error. */
    bool bPromptToRetryFailedShaderCompiles;
    /** Whether to log out shader job completion times on the worker thread.  Useful for tracking down which global shader is taking a long time. */
    bool bLogJobCompletionTimes;
    /** Target execution time for ProcessAsyncResults.  Larger values speed up async shader map processing but cause more hitchiness while async compiling is happening. */
    float ProcessGameThreadTargetTime;
    /** Base directory where temporary files are written out during multi core shader compiling. */
    FString ShaderBaseWorkingDirectory;
    /** Absolute version of ShaderBaseWorkingDirectory. */
    FString AbsoluteShaderBaseWorkingDirectory;
    /** Absolute path to the directory to dump shader debug info to. */
    FString AbsoluteShaderDebugInfoDirectory;
    /** Name of the shader worker application. */
    FString ShaderCompileWorkerName;
    /** Whether the SCW has crashed and we should fall back to calling the compiler dll's directly. */
    bool bFallBackToDirectCompiles;

    /** 
     * Tracks the total time that shader compile workers have been busy since startup.  
     * Useful for profiling the shader compile worker thread time.
     */
    double WorkersBusyTime;

    /** 
     * Tracks which opt-in shader platforms have their warnings suppressed.
     */
    uint64 SuppressedShaderPlatforms;

    /** Launches the worker, returns the launched process handle. */
    FProcHandle LaunchWorker(const FString& WorkingDirectory, uint32 ProcessId, uint32 ThreadId, const FString& WorkerInputFile, const FString& WorkerOutputFile);

    /** Blocks on completion of the given shader maps. */
    void BlockOnShaderMapCompletion(const TArray<int32>& ShaderMapIdsToFinishCompiling, TMap<int32, FShaderMapFinalizeResults>& CompiledShaderMaps);

    /** Blocks on completion of all shader maps. */
    void BlockOnAllShaderMapCompletion(TMap<int32, FShaderMapFinalizeResults>& CompiledShaderMaps);

    /** Finalizes the given shader map results and optionally assigns the affected shader maps to materials, while attempting to stay within an execution time budget. */
    void ProcessCompiledShaderMaps(TMap<int32, FShaderMapFinalizeResults>& CompiledShaderMaps, float TimeBudget);

    /** Finalizes the given Niagara shader map results and assigns the affected shader maps to Niagara scripts, while attempting to stay within an execution time budget. */
    void ProcessCompiledNiagaraShaderMaps(TMap<int32, FShaderMapFinalizeResults>& CompiledShaderMaps, float TimeBudget);

    /** Propagate the completed compile to primitives that might be using the materials compiled. */
    void PropagateMaterialChangesToPrimitives(const TMap<FMaterial*, class FMaterialShaderMap*>& MaterialsToUpdate);

    /** Recompiles shader jobs with errors if requested, and returns true if a retry was needed. */
    bool HandlePotentialRetryOnError(TMap<int32, FShaderMapFinalizeResults>& CompletedShaderMaps);

public:
    
    ENGINE_API FShaderCompilingManager();

    /** 
     * Returns whether to display a notification that shader compiling is happening in the background. 
     * Note: This is dependent on NumOutstandingJobs which is updated from another thread, so the results are non-deterministic.
     */
    bool ShouldDisplayCompilingNotification() const 
    { 
        // Heuristic based on the number of jobs outstanding
        return NumOutstandingJobs > 80 || CompileQueue.Num() > 80 || NumExternalJobs > 10;
    }

    bool AllowAsynchronousShaderCompiling() const 
    {
        return bAllowAsynchronousShaderCompiling;
    }

    /** 
     * Returns whether async compiling is happening.
     * Note: This is dependent on NumOutstandingJobs which is updated from another thread, so the results are non-deterministic.
     */
    bool IsCompiling() const
    {
        return NumOutstandingJobs > 0 || PendingFinalizeShaderMaps.Num() > 0 || CompileQueue.Num() > 0 || NumExternalJobs > 0;
    }

    /**
     * return true if we have shader jobs in any state
     * shader jobs are removed when they are applied to the gamethreadshadermap
     * accessable from gamethread
     */
    bool HasShaderJobs() const
    {
        return ShaderMapJobs.Num() > 0 || PendingFinalizeShaderMaps.Num() > 0;
    }

    /** 
     * Returns the number of outstanding compile jobs.
     * Note: This is dependent on NumOutstandingJobs which is updated from another thread, so the results are non-deterministic.
     */
    int32 GetNumRemainingJobs() const
    {
        return NumOutstandingJobs + NumExternalJobs;
    }

    void SetExternalJobs(int32 NumJobs)
    {
        NumExternalJobs = NumJobs;
    }

    ENGINE_API bool GetDumpShaderDebugInfo() const;

    const FString& GetAbsoluteShaderDebugInfoDirectory() const
    {
        return AbsoluteShaderDebugInfoDirectory;
    }

    bool AreWarningsSuppressed(const EShaderPlatform Platform) const
    {
        return (SuppressedShaderPlatforms & (static_cast<uint64>(1) << Platform)) != 0;
    }

    void SuppressWarnings(const EShaderPlatform Platform)
    {
        SuppressedShaderPlatforms |= static_cast<uint64>(1) << Platform;
    }

    /** 
     * Adds shader jobs to be asynchronously compiled. 
     * FinishCompilation or ProcessAsyncResults must be used to get the results.
     */
    ENGINE_API void AddJobs(TArray<FShaderCommonCompileJob*>& NewJobs, bool bApplyCompletedShaderMapForRendering, bool bOptimizeForLowLatency, bool bRecreateComponentRenderStateOnCompletion);

    /**
    * Removes all outstanding compile jobs for the passed shader maps.
    */
    ENGINE_API void CancelCompilation(const TCHAR* MaterialName, const TArray<int32>& ShaderMapIdsToCancel);

    /** 
     * Blocks until completion of the requested shader maps.  
     * This will not assign the shader map to any materials, the caller is responsible for that.
     */
    ENGINE_API void FinishCompilation(const TCHAR* MaterialName, const TArray<int32>& ShaderMapIdsToFinishCompiling);

    /** 
     * Blocks until completion of all async shader compiling, and assigns shader maps to relevant materials.
     * This should be called before exit if the DDC needs to be made up to date. 
     */
    ENGINE_API void FinishAllCompilation();


    /** 
     * Shutdown the shader compiler manager, this will shutdown immediately and not process any more shader compile requests. 
     */
    ENGINE_API void Shutdown();


    /** 
     * Processes completed asynchronous shader maps, and assigns them to relevant materials.
     * @param bLimitExecutionTime - When enabled, ProcessAsyncResults will be bandwidth throttled by ProcessGameThreadTargetTime, to limit hitching.
     *        ProcessAsyncResults will then have to be called often to finish all shader maps (eg from Tick).  Otherwise, all compiled shader maps will be processed.
     * @param bBlockOnGlobalShaderCompletion - When enabled, ProcessAsyncResults will block until global shader maps are complete.
     *        This must be done before using global shaders for rendering.
     */
    ENGINE_API void ProcessAsyncResults(bool bLimitExecutionTime, bool bBlockOnGlobalShaderCompletion);

    /**
     * Returns true if the given shader compile worker is still running.
     */
    static bool IsShaderCompilerWorkerRunning(FProcHandle & WorkerHandle);
};

/** The global shader compiling thread manager. */
extern ENGINE_API FShaderCompilingManager* GShaderCompilingManager;

/** The shader precompilers for each platform.  These are only set during the console shader compilation while cooking or in the PrecompileShaders commandlet. */
extern class FConsoleShaderPrecompiler* GConsoleShaderPrecompilers[SP_NumPlatforms];

/** Enqueues a shader compile job with GShaderCompilingManager. */
extern ENGINE_API void GlobalBeginCompileShader(
    const FString& DebugGroupName,
    class FVertexFactoryType* VFType,
    class FShaderType* ShaderType,
    const class FShaderPipelineType* ShaderPipelineType,
    const TCHAR* SourceFilename,
    const TCHAR* FunctionName,
    FShaderTarget Target,
    FShaderCompileJob* NewJob,
    TArray<FShaderCommonCompileJob*>& NewJobs,
    bool bAllowDevelopmentShaderCompile = true
    );

/** Implementation of the 'recompileshaders' console command.  Recompiles shaders at runtime based on various criteria. */
extern bool RecompileShaders(const TCHAR* Cmd, FOutputDevice& Ar);

/** Returns whether all global shader types containing the substring are complete and ready for rendering. if type name is null, check everything */
extern ENGINE_API bool IsGlobalShaderMapComplete(const TCHAR* TypeNameSubstring = nullptr);

/**
* Makes sure all global shaders are loaded and/or compiled for the passed in platform.
* Note: if compilation is needed, this only kicks off the compile.
*
* @param    Platform    Platform to verify global shaders for
*/
extern ENGINE_API void VerifyGlobalShaders(EShaderPlatform Platform, bool bLoadedFromCacheFile);

/**
* Forces a recompile of the global shaders.
*/
extern ENGINE_API void RecompileGlobalShaders();

/**
* Recompiles global shaders and material shaders
* rebuilds global shaders and also
* clears the cooked platform data for all materials if there is a global shader change detected
* can be slow
*/
extern ENGINE_API bool RecompileChangedShadersForPlatform(const FString& PlatformName);

/**
* Begins recompiling the specified global shader types, and flushes their bound shader states.
* FinishRecompileGlobalShaders must be called after this and before using the global shaders for anything.
*/
extern ENGINE_API void BeginRecompileGlobalShaders(const TArray<FShaderType*>& OutdatedShaderTypes, const TArray<const FShaderPipelineType*>& OutdatedShaderPipelineTypes, EShaderPlatform ShaderPlatform);

/** Finishes recompiling global shaders.  Must be called after BeginRecompileGlobalShaders. */
extern ENGINE_API void FinishRecompileGlobalShaders();

/** Called by the shader compiler to process completed global shader jobs. */
extern ENGINE_API void ProcessCompiledGlobalShaders(const TArray<FShaderCommonCompileJob*>& CompilationResults);

/**
* Saves the global shader map as a file for the target platform.
* @return the name of the file written
*/
extern ENGINE_API FString SaveGlobalShaderFile(EShaderPlatform Platform, FString SavePath, class ITargetPlatform* TargetPlatform = nullptr);

/**
* Recompiles global shaders
*
* @param PlatformName                    Name of the Platform the shaders are compiled for
* @param OutputDirectory                The directory the compiled data will be stored to
* @param MaterialsToLoad                List of Materials that need to be loaded and compiled
* @param SerializedShaderResources        Serialized shader resources
* @param MeshMaterialMaps                Mesh material maps
* @param ModifiedFiles                    Returns the list of modified files if not NULL
* @param bCompileChangedShaders        Whether to compile all changed shaders or the specific material that is passed
**/
extern ENGINE_API void RecompileShadersForRemote(
    const FString& PlatformName,
    EShaderPlatform ShaderPlatform,
    const FString& OutputDirectory,
    const TArray<FString>& MaterialsToLoad,
    const TArray<uint8>& SerializedShaderResources,
    TArray<uint8>* MeshMaterialMaps,
    TArray<FString>* ModifiedFiles,
    bool bCompileChangedShaders = true);

extern ENGINE_API void CompileGlobalShaderMap(bool bRefreshShaderMap=false);
extern ENGINE_API void CompileGlobalShaderMap(EShaderPlatform Platform, bool bRefreshShaderMap = false);
extern ENGINE_API void CompileGlobalShaderMap(ERHIFeatureLevel::Type InFeatureLevel, bool bRefreshShaderMap=false);

extern ENGINE_API FString GetGlobalShaderMapDDCKey();

extern ENGINE_API FString GetMaterialShaderMapDDCKey();

shaderCompile.cpp文件，主要按照xge的搞，另外，转换为绝对路径的代码删掉：

 

 

 

 

 

 

 

 

 

 shaderCompilerFASTBuild.cpp:

// Copyright 1998-2019 Epic Games, Inc. All Rights Reserved.

#include "ShaderCompiler.h"
#include "GenericPlatform/GenericPlatformFile.h"
#include "HAL/PlatformFilemanager.h"
#include "HAL/FileManager.h"
#include "Misc/ScopeLock.h"
#include "CoreGlobals.h"
#include "Misc/ConfigCacheIni.h"

#if PLATFORM_WINDOWS

// --------------------------------------------------------------------------
//                          Legacy XGE Xml interface                         
// --------------------------------------------------------------------------
namespace FASTBuildConsoleVariables
{
    /** The total number of batches to fill with shaders before creating another group of batches. */
    int32 BatchGroupSize = 128;
    FAutoConsoleVariableRef CVarFASTBuildShaderCompileBatchGroupSize(
        TEXT("r.FASTBuildShaderCompile.BatchGroupSize"),
        BatchGroupSize,
        TEXT("Specifies the number of batches to fill with shaders.\n")
        TEXT("Shaders are spread across this number of batches until all the batches are full.\n")
        TEXT("This allows the FASTBuild compile to go wider when compiling a small number of shaders.\n")
        TEXT("Default = 128\n"),
        ECVF_Default);

    int32 Enabled = 1;
    FAutoConsoleVariableRef CVarFASTBuildShaderCompile(
        TEXT("r.FASTBuildShaderCompile"),
        Enabled,
        TEXT("Enables or disables the use of FASTBuild to build shaders.\n")
        TEXT("0: Local builds only. \n")
        TEXT("1: Distribute builds using FASTBuild."),
        ECVF_Default);

    /** The maximum number of shaders to group into a single FASTBuild task. */
    int32 BatchSize = 16;
    FAutoConsoleVariableRef CVarFASTBuildShaderCompileBatchSize(
        TEXT("r.FASTBuildShaderCompile.BatchSize"),
        BatchSize,
        TEXT("Specifies the number of shaders to batch together into a single FASTBuild task.\n")
        TEXT("Default = 16\n"),
        ECVF_Default);

    /**
    * The number of seconds to wait after a job is submitted before kicking off the XGE process.
    * This allows time for the engine to enqueue more shaders, so we get better batching.
    */
    float JobTimeout = 0.5f;
    FAutoConsoleVariableRef CVarFASTBuildShaderCompileJobTimeout(
        TEXT("r.FastBuiShaderCompile.Xml.JobTimeout"),
        JobTimeout,
        TEXT("The number of seconds to wait for additional shader jobs to be submitted before starting a build.\n")
        TEXT("Default = 0.5\n"),
        ECVF_Default);
}

static FString GetFASTBuild_ExecutablePath()
{
    return FPaths::ConvertRelativePathToFull(FPaths::EngineDir()) / TEXT("Binaries\\ThirdParty\\FastBuild\\FBuild.exe");
}

static FString FASTBuild_CachePath(TEXT("..\\Saved\\FASTBuildCache"));
static const FString FASTBuild_OrbisSDKToolchainRoot(TEXT("Engine\\Binaries\\ThirdParty\\PS4\\OrbisSDK"));
static const FString FASTBuild_Toolchain[]
{
    TEXT("Engine\\Binaries\\ThirdParty\\Windows\\DirectX\\x64\\d3dcompiler_47.dll"),
    /*TEXT("Engine\\Binaries\\Win64\\dxcompiler.dll"),
    TEXT("Engine\\Binaries\\ThirdParty\\Windows\\DirectX\\x64\\WinPixEventRuntime.dll"),
    TEXT("Engine\\Binaries\\Win64\\dxil.dll"),*/
};


static const FString FASTBuild_ScriptFileName(TEXT("fastbuildscript.bff"));
static const FString FASTBuild_InputFileName(TEXT("Worker.in"));
static const FString FASTBuild_OutputFileName(TEXT("Worker.out"));
static const FString FASTBuild_SuccessFileName(TEXT("Success"));

bool FShaderCompileFASTBuildThreadRunnable::IsSupported()
{
    // Check to see if the FASTBuild exe exists.
    if (FASTBuildConsoleVariables::Enabled == 1)
    {
        IPlatformFile& PlatformFile = FPlatformFileManager::Get().GetPlatformFile();
        if (!PlatformFile.FileExists(*GetFASTBuild_ExecutablePath()))
        {
            UE_LOG(LogShaderCompilers, Warning, TEXT("Cannot use FASTBuild Shader Compiler as FASTBuild is not found. Target Path:%s"), *GetFASTBuild_ExecutablePath());
            FASTBuildConsoleVariables::Enabled = 0;
        }
        else
        {
            UE_LOG(LogShaderCompilers, Warning, TEXT("FASTBuild Shader Compiler found. Target Path:%s"), *GetFASTBuild_ExecutablePath());
        }
    }
    return FASTBuildConsoleVariables::Enabled == 1;

    /*if (FASTBuildShaderCompilerVariables::Enabled == 1)
    {

        FString BaseRoot = FPaths::RootDir();
        const auto DirectoryMappings = AllShaderSourceDirectoryMappings();
        for (const auto& MappingEntry : DirectoryMappings)
        {
            if (!GetCommonBaseDir(MappingEntry.Value, BaseRoot, BaseRoot))
            {
                UE_LOG(LogShaderCompilers, Warning, TEXT("Con not Use FASTBuild to compile shader,\
                                        Because the shader files are not in the same Hardware Disk Drive!"));
            }
        }
    }

    return FASTBuildShaderCompilerVariables::Enabled == 1;*/
}


/** Initialization constructor. */
FShaderCompileFASTBuildThreadRunnable::FShaderCompileFASTBuildThreadRunnable(class FShaderCompilingManager* InManager)
    : FShaderCompileThreadRunnableBase(InManager)
    , BuildProcessID(INDEX_NONE)
    , ShaderBatchesInFlightCompleted(0)
    , FASTBuildWorkingDirectory(InManager->AbsoluteShaderBaseWorkingDirectory / TEXT("FASTBuild"))
    , FASTBuildDirectoryIndex(0)
    , LastAddTime(0)
    , StartTime(0)
    , BatchIndexToCreate(0)
    , BatchIndexToFill(0)
{
    //FString Path = FPaths::EngineDir() / TEXT("Binaries/Win64/ShaderCompileWorker-Win64-Debug.exe");
//    InManager->ShaderCompileWorkerName = FPaths::ConvertRelativePathToFull(Path);
    //FbWorkingDirectory = InManager->AbsoluteShaderBaseWorkingDirectory / TEXT("FASTBuild_Shader");
}

FShaderCompileFASTBuildThreadRunnable::~FShaderCompileFASTBuildThreadRunnable()
{
    if (BuildProcessHandle.IsValid())
    {
        // We still have a build in progress.
        // Kill it...
        FPlatformProcess::TerminateProc(BuildProcessHandle);
        FPlatformProcess::CloseProc(BuildProcessHandle);
    }

    // Clean up any intermediate files/directories we've got left over.
    IFileManager::Get().DeleteDirectory(*FASTBuildWorkingDirectory, false, true);

    // Delete all the shader batch instances we have.
    for (FShaderBatch* Batch : ShaderBatchesIncomplete)
        delete Batch;

    for (FShaderBatch* Batch : ShaderBatchesInFlight)
        delete Batch;

    for (FShaderBatch* Batch : ShaderBatchesFull)
        delete Batch;

    ShaderBatchesIncomplete.Empty();
    ShaderBatchesInFlight.Empty();
    ShaderBatchesFull.Empty();
}


void FShaderCompileFASTBuildThreadRunnable::PostCompletedJobsForBatch(FShaderBatch* Batch)
{
    // Enter the critical section so we can access the input and output queues
    FScopeLock Lock(&Manager->CompileQueueSection);
    for (FShaderCommonCompileJob* Job : Batch->GetJobs())
    {
        FShaderMapCompileResults& ShaderMapResults = Manager->ShaderMapJobs.FindChecked(Job->Id);
        ShaderMapResults.FinishedJobs.Add(Job);
        ShaderMapResults.bAllJobsSucceeded = ShaderMapResults.bAllJobsSucceeded && Job->bSucceeded;
    }

    // Using atomics to update NumOutstandingJobs since it is read outside of the critical section
    FPlatformAtomics::InterlockedAdd(&Manager->NumOutstandingJobs, -Batch->NumJobs());
}

void FShaderCompileFASTBuildThreadRunnable::FShaderBatch::AddJob(FShaderCommonCompileJob* Job)
{
    // We can only add jobs to a batch which hasn't been written out yet.
    if (bTransferFileWritten)
    {
        UE_LOG(LogShaderCompilers, Fatal, TEXT("Attempt to add shader compile jobs to a FASTBuild shader batch which has already been written to disk."));
    }
    else
    {
        Jobs.Add(Job);
    }
}

void FShaderCompileFASTBuildThreadRunnable::FShaderBatch::WriteTransferFile()
{
    // Write out the file that the worker app is waiting for, which has all the information needed to compile the shader.
    FArchive* TransferFile = FShaderCompileUtilities::CreateFileHelper(InputFileNameAndPath);
    FShaderCompileUtilities::DoWriteTasks(Jobs, *TransferFile);
    delete TransferFile;

    bTransferFileWritten = true;
}

void FShaderCompileFASTBuildThreadRunnable::FShaderBatch::SetIndices(int32 InDirectoryIndex, int32 InBatchIndex)
{
    DirectoryIndex = InDirectoryIndex;
    BatchIndex = InBatchIndex;

    WorkingDirectory = FString::Printf(TEXT("%s/%d/%d"), *DirectoryBase, DirectoryIndex, BatchIndex);

    InputFileNameAndPath = WorkingDirectory / InputFileName;
    OutputFileNameAndPath = WorkingDirectory / OutputFileName;
    SuccessFileNameAndPath = WorkingDirectory / SuccessFileName;
}

void FShaderCompileFASTBuildThreadRunnable::FShaderBatch::CleanUpFiles(bool keepInputFile)
{
    if (!keepInputFile)
    {
        FShaderCompileUtilities::DeleteFileHelper(InputFileNameAndPath);
    }

    FShaderCompileUtilities::DeleteFileHelper(OutputFileNameAndPath);
    FShaderCompileUtilities::DeleteFileHelper(SuccessFileNameAndPath);
}

static void FASTBuildWriteScriptFileHeader(FArchive* ScriptFile, const FString& WorkerName)
{
    static const TCHAR HeaderTemplate[] =
        TEXT("Settings\r\n")
        TEXT("{\r\n")
        TEXT("\t.CachePath = '%s'\r\n")
        TEXT("}\r\n")
        TEXT("\r\n")
        TEXT("Compiler('ShaderCompiler')\r\n")
        TEXT("{\r\n")
        TEXT("\t.CompilerFamily = 'custom'\r\n")
        TEXT("\t.Executable = '%s'\r\n")
        TEXT("\t.ExecutableRootPath = '%s'\r\n")
        TEXT("\t.SimpleDistributionMode = true\r\n")
        TEXT("\t.CustomEnvironmentVariables = { 'SCE_ORBIS_SDK_DIR=%%1%s' }\r\n")
        TEXT("\t.ExtraFiles = \r\n")
        TEXT("\t{\r\n");


    FString HeaderString = FString::Printf(HeaderTemplate, *FASTBuild_CachePath, *WorkerName, *IFileManager::Get().ConvertToAbsolutePathForExternalAppForRead(*FPaths::RootDir()), *FASTBuild_OrbisSDKToolchainRoot);
    ScriptFile->Serialize((void*)StringCast<ANSICHAR>(*HeaderString, HeaderString.Len()).Get(), sizeof(ANSICHAR) * HeaderString.Len());

    for (const FString& ExtraFilePartialPath : FASTBuild_Toolchain)
    {
        FString ExtraFile = TEXT("\t\t'") + IFileManager::Get().ConvertToAbsolutePathForExternalAppForRead(*(FPaths::RootDir() / ExtraFilePartialPath)) + TEXT("',\r\n");
        ScriptFile->Serialize((void*)StringCast<ANSICHAR>(*ExtraFile, ExtraFile.Len()).Get(), sizeof(ANSICHAR) * ExtraFile.Len());
    }

    class FDependencyEnumerator:public IPlatformFile::FDirectoryVisitor
    {
    public:
        FDependencyEnumerator(FArchive* InScriptFile, const TCHAR* InPrefix, const TCHAR* InExtension
            //, const TCHAR* InExcludeExtensions = NULL
        )
            :ScriptFile(InScriptFile)
            , Prefix(InPrefix)
            , Extension(InExtension)
        {
        }

        virtual bool Visit(const TCHAR* FilenameChar, bool bIsDirectory) override
        {

            if (!bIsDirectory)
            {
                FString Filename = FString(FilenameChar);

                if ((!Prefix || Filename.Contains(Prefix)) && (!Extension || Filename.EndsWith(Extension)))
                {
                    FString ExtraFile = TEXT("\t\t'") + IFileManager::Get().ConvertToAbsolutePathForExternalAppForWrite(*Filename) + TEXT("',\r\n");
                    ScriptFile->Serialize((void*)StringCast<ANSICHAR>(*ExtraFile, ExtraFile.Len()).Get(), sizeof(ANSICHAR) * ExtraFile.Len());
                }
            }
            return true;
        }

        FArchive* const ScriptFile;
        const TCHAR* Prefix;
        const TCHAR* Extension;
    };

    FDependencyEnumerator DllDeps = FDependencyEnumerator(ScriptFile, TEXT("ShaderCompileWorker-"), TEXT(".dll"));
    IFileManager::Get().IterateDirectoryRecursively(*FPlatformProcess::GetModulesDirectory(), DllDeps);

    FDependencyEnumerator ManifestDeps = FDependencyEnumerator(ScriptFile, TEXT("ShaderCompileWorker"), TEXT(".modules"));
    IFileManager::Get().IterateDirectoryRecursively(*FPlatformProcess::GetModulesDirectory(), ManifestDeps);

    FDependencyEnumerator ShaderUshDeps = FDependencyEnumerator(ScriptFile, nullptr, TEXT(".ush"));
    IFileManager::Get().IterateDirectoryRecursively(FPlatformProcess::ShaderDir(), ShaderUshDeps);

    FDependencyEnumerator ShaderUsfDeps = FDependencyEnumerator(ScriptFile, nullptr, TEXT(".usf"));
    IFileManager::Get().IterateDirectoryRecursively(FPlatformProcess::ShaderDir(), ShaderUsfDeps);

    /*FDependencyEnumerator ShaderHDeps = FDependencyEnumerator(ScriptFile, nullptr, TEXT(".h"));
    IFileManager::Get().IterateDirectoryRecursively(FPlatformProcess::ShaderDir(), ShaderHDeps);

    FDependencyEnumerator ShaderHlslDeps = FDependencyEnumerator(ScriptFile, nullptr, TEXT(".hlsl"));
    IFileManager::Get().IterateDirectoryRecursively(FPlatformProcess::ShaderDir(), ShaderHlslDeps);

    FDependencyEnumerator ShaderGlslDeps = FDependencyEnumerator(ScriptFile, nullptr, TEXT(".glsl"));
    IFileManager::Get().IterateDirectoryRecursively(FPlatformProcess::ShaderDir(), ShaderGlslDeps);*/

    FDependencyEnumerator PluginShaderUshDeps = FDependencyEnumerator(ScriptFile, nullptr, TEXT(".ush"));
    IFileManager::Get().IterateDirectoryRecursively(*FPaths::Combine(*(FPaths::EngineDir()), TEXT("Plugins")), PluginShaderUshDeps);

    FDependencyEnumerator PluginShaderUsfDeps = FDependencyEnumerator(ScriptFile, nullptr, TEXT(".usf"));
    IFileManager::Get().IterateDirectoryRecursively(*FPaths::Combine(*(FPaths::EngineDir()), TEXT("Plugins")), PluginShaderUsfDeps);

    /*FDependencyEnumerator PluginShaderHDeps = FDependencyEnumerator(ScriptFile, nullptr, TEXT(".h"));
    IFileManager::Get().IterateDirectoryRecursively(*FPaths::Combine(*(FPaths::EngineDir()), TEXT("Plugins")), PluginShaderHDeps);

    FDependencyEnumerator PluginShaderHlslDeps = FDependencyEnumerator(ScriptFile, nullptr, TEXT(".hlsl"));
    IFileManager::Get().IterateDirectoryRecursively(*FPaths::Combine(*(FPaths::EngineDir()), TEXT("Plugins")), PluginShaderHlslDeps);

    FDependencyEnumerator PluginShaderGlslDeps = FDependencyEnumerator(ScriptFile, nullptr, TEXT(".glsl"));
    IFileManager::Get().IterateDirectoryRecursively(*FPaths::Combine(*(FPaths::EngineDir()), TEXT("Plugins")), PluginShaderGlslDeps);*/

    const FString ExtraFilesFooter =
        TEXT("\t}\r\n")
        TEXT("}\r\n");
    ScriptFile->Serialize((void*)StringCast<ANSICHAR>(*ExtraFilesFooter, ExtraFilesFooter.Len()).Get(), sizeof(ANSICHAR) * ExtraFilesFooter.Len());

}

static void WriteFASTBuildScriptFileFooter(FArchive* ScriptFile)
{
}


void FShaderCompileFASTBuildThreadRunnable::GatherResultsFromFASTBuild()
{
    IPlatformFile& PlatformFile = FPlatformFileManager::Get().GetPlatformFile();
    IFileManager& FileManager = IFileManager::Get();

    // Reverse iterate so we can remove batches that have completed as we go.
    for (int32 Index = ShaderBatchesInFlight.Num() - 1; Index >= 0; Index--)
    {
        FShaderBatch* Batch = ShaderBatchesInFlight[Index];

        // If this batch is completed already, skip checks.
        if (Batch->bSuccessfullyCompleted)
        {
            continue;
        }

        // Instead of checking for another file, we just check to see if the file exists, and if it does, we check if it has a write lock on it. FASTBuild never does partial writes, so this should tell us if FASTBuild is complete.
        // Perform the same checks on the worker output file to verify it came from this build.
        if (PlatformFile.FileExists(*Batch->OutputFileNameAndPath))
        {
            if (FileManager.FileSize(*Batch->OutputFileNameAndPath) > 0)
            {
                IFileHandle* Handle = PlatformFile.OpenWrite(*Batch->OutputFileNameAndPath, true);
                if (Handle)
                {
                    delete Handle;

                    if (PlatformFile.GetTimeStamp(*Batch->OutputFileNameAndPath) >= ScriptFileCreationTime)
                    {
                        FArchive* OutputFilePtr = FileManager.CreateFileReader(*Batch->OutputFileNameAndPath, FILEREAD_Silent);
                        if (OutputFilePtr)
                        {
                            FArchive& OutputFile = *OutputFilePtr;
                            FShaderCompileUtilities::DoReadTaskResults(Batch->GetJobs(), OutputFile);

                            // Close the output file.
                            delete OutputFilePtr;

                            // Cleanup the worker files
                            // Do NOT clean up files until the whole batch is done, so we can clean them all up once the fastbuild process exits. Otherwise there is a race condition between FastBuild checking the output files, and us deleting them here.
                            //Batch->CleanUpFiles(false);            // (false = don't keep the input file)
                            Batch->bSuccessfullyCompleted = true;
                            PostCompletedJobsForBatch(Batch);
                            //ShaderBatchesInFlight.RemoveAt(Index);
                            ShaderBatchesInFlightCompleted++;
                            //delete Batch;
                        }
                    }
                }
            }
        }
    }
}


int32 FShaderCompileFASTBuildThreadRunnable::CompilingLoop()
{

    bool bWorkRemaining = false;

    // We can only run one XGE build at a time.
    // Check if a build is currently in progress.
    if (BuildProcessHandle.IsValid())
    {
        // Read back results from the current batches in progress.
        GatherResultsFromFASTBuild();

        bool bDoExitCheck = false;
        if (FPlatformProcess::IsProcRunning(BuildProcessHandle))
        {
            if (ShaderBatchesInFlight.Num() == ShaderBatchesInFlightCompleted)
            {
                // We've processed all batches.
                // Wait for the XGE console process to exit
                FPlatformProcess::WaitForProc(BuildProcessHandle);
                bDoExitCheck = true;
            }
        }
        else
        {
            bDoExitCheck = true;
        }

        if (bDoExitCheck)
        {
            if (ShaderBatchesInFlight.Num() > ShaderBatchesInFlightCompleted)
            {
                // The build process has stopped.
                // Do one final pass over the output files to gather any remaining results.
                GatherResultsFromFASTBuild();
            }

            // The build process is no longer running.
            // We need to check the return code for possible failure
            int32 ReturnCode = 0;
            FPlatformProcess::GetProcReturnCode(BuildProcessHandle, &ReturnCode);

            switch (ReturnCode)
            {
            case 0:
                // No error
                break;

            case 1:
                // One or more of the shader compile worker processes crashed.
                UE_LOG(LogShaderCompilers, Fatal, TEXT("An error occurred during an XGE shader compilation job. One or more of the shader compile worker processes exited unexpectedly (Code 1)."));
                break;

            case 2:
                // Fatal IncrediBuild error
                UE_LOG(LogShaderCompilers, Fatal, TEXT("An error occurred during an FASTBuild shader compilation job. XGConsole.exe returned a fatal Incredibuild error (Code 2)."));
                break;

            case 3:
                // User canceled the build
                UE_LOG(LogShaderCompilers, Display, TEXT("The user terminated an XGE shader compilation job. Incomplete shader jobs will be redispatched in another FASTBuild build."));
                break;

            default:
                UE_LOG(LogShaderCompilers, Display, TEXT("An unknown error occurred during an XGE shader compilation job (Code %d). Incomplete shader jobs will be redispatched in another FASTBuild build."), ReturnCode);
                break;
            }

            // Reclaim jobs from the workers which did not succeed (if any).
            for (int i = 0; i < ShaderBatchesInFlight.Num(); ++i)
            {
                FShaderBatch* Batch = ShaderBatchesInFlight[i];

                if (Batch->bSuccessfullyCompleted)
                {
                    // If we completed successfully, clean up.
                    //PostCompletedJobsForBatch(Batch);
                    Batch->CleanUpFiles(false);

                    // This will be a dangling pointer until we clear the array at the end of this for loop
                    delete Batch;
                }
                else
                {

                    // Delete any output/success files, but keep the input file so we don't have to write it out again.
                    Batch->CleanUpFiles(true);

                    // We can't add any jobs to a shader batch which has already been written out to disk,
                    // so put the batch back into the full batches list, even if the batch isn't full.
                    ShaderBatchesFull.Add(Batch);

                    // Reset the batch/directory indices and move the input file to the correct place.
                    FString OldInputFilename = Batch->InputFileNameAndPath;
                    Batch->SetIndices(FASTBuildDirectoryIndex, BatchIndexToCreate++);
                    FShaderCompileUtilities::MoveFileHelper(Batch->InputFileNameAndPath, OldInputFilename);
                }
            }
            ShaderBatchesInFlightCompleted = 0;
            ShaderBatchesInFlight.Empty();
            FPlatformProcess::CloseProc(BuildProcessHandle);
        }

        bWorkRemaining |= ShaderBatchesInFlight.Num() > ShaderBatchesInFlightCompleted;
    }
    // No build process running. Check if we can kick one off now.
    else
    {
        // Determine if enough time has passed to allow a build to kick off.
        // Since shader jobs are added to the shader compile manager asynchronously by the engine, 
        // we want to give the engine enough time to queue up a large number of shaders.
        // Otherwise we will only be kicking off a small number of shader jobs at once.
        bool BuildDelayElapsed = (((FPlatformTime::Cycles() - LastAddTime) * FPlatformTime::GetSecondsPerCycle()) >= FASTBuildConsoleVariables::JobTimeout);
        bool HasJobsToRun = (ShaderBatchesIncomplete.Num() > 0 || ShaderBatchesFull.Num() > 0);

        if (BuildDelayElapsed && HasJobsToRun && ShaderBatchesInFlight.Num() == ShaderBatchesInFlightCompleted)
        {
            // Move all the pending shader batches into the in-flight list.
            ShaderBatchesInFlight.Reserve(ShaderBatchesIncomplete.Num() + ShaderBatchesFull.Num());

            for (FShaderBatch* Batch : ShaderBatchesIncomplete)
            {
                // Check we've actually got jobs for this batch.
                check(Batch->NumJobs() > 0);

                // Make sure we've written out the worker files for any incomplete batches.
                Batch->WriteTransferFile();
                ShaderBatchesInFlight.Add(Batch);
            }

            for (FShaderBatch* Batch : ShaderBatchesFull)
            {
                // Check we've actually got jobs for this batch.
                check(Batch->NumJobs() > 0);

                ShaderBatchesInFlight.Add(Batch);
            }

            ShaderBatchesFull.Empty();
            ShaderBatchesIncomplete.Empty(FASTBuildConsoleVariables::BatchGroupSize);

            FString ScriptFilename = FASTBuildWorkingDirectory / FString::FromInt(FASTBuildDirectoryIndex) / FASTBuild_ScriptFileName;
            UE_LOG(LogShaderCompilers, Log, TEXT("Script Path:%s"), *ScriptFilename);
            // Create the XGE script file.
            FArchive* ScriptFile = FShaderCompileUtilities::CreateFileHelper(ScriptFilename);
            FASTBuildWriteScriptFileHeader(ScriptFile, Manager->ShaderCompileWorkerName);

            // Write the XML task line for each shader batch
            for (FShaderBatch* Batch : ShaderBatchesInFlight)
            {
                FString WorkerAbsoluteDirectory = IFileManager::Get().ConvertToAbsolutePathForExternalAppForWrite(*Batch->WorkingDirectory);
                FPaths::NormalizeDirectoryName(WorkerAbsoluteDirectory);

                // Proper path should be "WorkingDir" 0 0 "Worker.in" "Worker.out"

                FString ExecFunction = FString::Printf(
                    TEXT("ObjectList('ShaderBatch-%d')\r\n")
                    TEXT("{\r\n")
                    TEXT("\t.Compiler = 'ShaderCompiler'\r\n")
                    TEXT("\t.CompilerOptions = '\"\" %d %d \"%%1\" \"%%2\"' \r\n")
                    TEXT("\t.CompilerOutputExtension = '.out' \r\n")
                    TEXT("\t.CompilerInputFiles = {'%s'}\r\n")
                    TEXT("\t.CompilerOutputPath = '%s'\r\n")
                    TEXT("}\r\n\r\n"),
                    Batch->BatchIndex,
                    Manager->ProcessId,
                    Batch->BatchIndex,
                    *Batch->InputFileNameAndPath,
                    *WorkerAbsoluteDirectory

                );

                ScriptFile->Serialize((void*)StringCast<ANSICHAR>(*ExecFunction, ExecFunction.Len()).Get(), sizeof(ANSICHAR) * ExecFunction.Len());
            }//end for

            FString AliasBuildTargetOpen = FString(
                TEXT("Alias('all')\r\n")
                TEXT("{\r\n")
                TEXT("\t.Targets = { \r\n")
            );

            ScriptFile->Serialize((void*)StringCast<ANSICHAR>(*AliasBuildTargetOpen, AliasBuildTargetOpen.Len()).Get(),
                sizeof(ANSICHAR) * AliasBuildTargetOpen.Len());


            for (FShaderBatch* Batch : ShaderBatchesInFlight)
            {
                FString TargetExport =
                    FString::Printf(TEXT("'ShaderBatch-%d', "), Batch->BatchIndex);

                ScriptFile->Serialize((void*)StringCast<ANSICHAR>(*TargetExport, TargetExport.Len()).Get(),
                    sizeof(ANSICHAR) * TargetExport.Len());

            }

            FString AliasBuildTargetClose = FString(TEXT(" }\r\n}\r\n"));
            ScriptFile->Serialize((void*)StringCast<ANSICHAR>(*AliasBuildTargetClose, AliasBuildTargetClose.Len()).Get(),
                sizeof(ANSICHAR) * AliasBuildTargetClose.Len());

            // End the XML script file and close it.
            WriteFASTBuildScriptFileFooter(ScriptFile);//FASTBuildWriteScriptFileFooter  ;todo
            delete ScriptFile;
            ScriptFile = nullptr;

            // Grab the timestamp from the script file.
            // We use this to ignore any left over files from previous builds by only accepting files created after the script file.
            ScriptFileCreationTime = IFileManager::Get().GetTimeStamp(*ScriptFilename);

            StartTime = FPlatformTime::Cycles();

            // Use stop on errors so we can respond to shader compile worker crashes immediately.
            // Regular shader compilation errors are not returned as worker errors.
            FString FBConsoleArgs = TEXT("-config \"") + ScriptFilename + TEXT("\" -dist -monitor -cache");

            // Kick off the FASTBuild process...
            BuildProcessHandle = FPlatformProcess::CreateProc(*GetFASTBuild_ExecutablePath(), *FBConsoleArgs, false, false, true, &BuildProcessID, 0, nullptr, nullptr);
            if (!BuildProcessHandle.IsValid())
            {
                UE_LOG(LogShaderCompilers, Fatal, TEXT("Failed to launch %s during shader compilation."), *GetFASTBuild_ExecutablePath());
            }

            // If the engine crashes, we don't get a chance to kill the build process.
            // Start up the build monitor process to monitor for engine crashes.
            uint32 BuildMonitorProcessID;
            FProcHandle BuildMonitorHandle = FPlatformProcess::CreateProc(*Manager->ShaderCompileWorkerName,
                *FString::Printf(TEXT("-xgemonitor %d %d"), Manager->ProcessId, BuildProcessID),
                true, false, false, &BuildMonitorProcessID, 0, nullptr, nullptr);

            FPlatformProcess::CloseProc(BuildMonitorHandle);

            // Reset batch counters and switch directories
            BatchIndexToFill = 0;
            BatchIndexToCreate = 0;
            FASTBuildDirectoryIndex = 1 - FASTBuildDirectoryIndex;

            bWorkRemaining = true;
        }
    }

    // Try to prepare more shader jobs (even if a build is in flight).
    TArray<FShaderCommonCompileJob*> JobQueue;
    {
        // Enter the critical section so we can access the input and output queues
        FScopeLock Lock(&Manager->CompileQueueSection);

        // Grab as many jobs from the job queue as we can.
        int32 NumNewJobs = Manager->CompileQueue.Num();
        if (NumNewJobs > 0)
        {
            int32 DestJobIndex = JobQueue.AddUninitialized(NumNewJobs);
            for (int32 SrcJobIndex = 0; SrcJobIndex < NumNewJobs; SrcJobIndex++, DestJobIndex++)
            {
                JobQueue[DestJobIndex] = Manager->CompileQueue[SrcJobIndex];
            }

            Manager->CompileQueue.RemoveAt(0, NumNewJobs);
        }
    }

    if (JobQueue.Num() > 0)
    {
        // We have new jobs in the queue.
        // Group the jobs into batches and create the worker input files.
        for (int32 JobIndex = 0; JobIndex < JobQueue.Num(); JobIndex++)
        {
            if (BatchIndexToFill >= ShaderBatchesIncomplete.GetMaxIndex() || !ShaderBatchesIncomplete.IsAllocated(BatchIndexToFill))
            {
                // There are no more incomplete shader batches available.
                // Create another one...
                ShaderBatchesIncomplete.Insert(BatchIndexToFill, new FShaderBatch(
                    FASTBuildWorkingDirectory,
                    FASTBuild_InputFileName,
                    FASTBuild_SuccessFileName,
                    FASTBuild_OutputFileName,
                    FASTBuildDirectoryIndex,
                    BatchIndexToCreate));

                BatchIndexToCreate++;
            }

            // Add a single job to this batch
            FShaderBatch* CurrentBatch = ShaderBatchesIncomplete[BatchIndexToFill];
            CurrentBatch->AddJob(JobQueue[JobIndex]);

            // If the batch is now full...
            if (CurrentBatch->NumJobs() == FASTBuildConsoleVariables::BatchSize)
            {
                CurrentBatch->WriteTransferFile();

                // Move the batch to the full list.
                ShaderBatchesFull.Add(CurrentBatch);
                ShaderBatchesIncomplete.RemoveAt(BatchIndexToFill);
            }

            BatchIndexToFill++;
            BatchIndexToFill %= FASTBuildConsoleVariables::BatchGroupSize;
        }

        // Keep track of the last time we added jobs.
        LastAddTime = FPlatformTime::Cycles();

        bWorkRemaining = true;
    }

    if (Manager->bAllowAsynchronousShaderCompiling)
    {
        // Yield for a short while to stop this thread continuously polling the disk.
        FPlatformProcess::Sleep(0.01f);
    }

    return bWorkRemaining ? 1 : 0;
}

FArchive* FShaderCompileUtilities::CreateFileHelper(const FString& Filename)
{
    // TODO: This logic came from FShaderCompileThreadRunnable::WriteNewTasks().
    // We can't avoid code duplication unless we refactored the local worker too.

    FArchive* File = nullptr;
    int32 RetryCount = 0;
    // Retry over the next two seconds if we can't write out the file.
    // Anti-virus and indexing applications can interfere and cause this to fail.
    while (File == nullptr && RetryCount < 200)
    {
        if (RetryCount > 0)
        {
            FPlatformProcess::Sleep(0.01f);
        }
        File = IFileManager::Get().CreateFileWriter(*Filename, FILEWRITE_EvenIfReadOnly);
        RetryCount++;
    }
    if (File == nullptr)
    {
        File = IFileManager::Get().CreateFileWriter(*Filename, FILEWRITE_EvenIfReadOnly | FILEWRITE_NoFail);
    }
    checkf(File, TEXT("Failed to create file %s!"), *Filename);
    return File;
}

void FShaderCompileUtilities::MoveFileHelper(const FString& To, const FString& From)
{
    IPlatformFile& PlatformFile = FPlatformFileManager::Get().GetPlatformFile();

    if (PlatformFile.FileExists(*From))
    {
        FString DirectoryName;
        int32 LastSlashIndex;
        if (To.FindLastChar('/', LastSlashIndex))
        {
            DirectoryName = To.Left(LastSlashIndex);
        }
        else
        {
            DirectoryName = To;
        }

        // TODO: This logic came from FShaderCompileThreadRunnable::WriteNewTasks().
        // We can't avoid code duplication unless we refactored the local worker too.

        bool Success = false;
        int32 RetryCount = 0;
        // Retry over the next two seconds if we can't move the file.
        // Anti-virus and indexing applications can interfere and cause this to fail.
        while (!Success && RetryCount < 200)
        {
            if (RetryCount > 0)
            {
                FPlatformProcess::Sleep(0.01f);
            }

            // MoveFile does not create the directory tree, so try to do that now...
            Success = PlatformFile.CreateDirectoryTree(*DirectoryName);
            if (Success)
            {
                Success = PlatformFile.MoveFile(*To, *From);
            }
            RetryCount++;
        }
        checkf(Success, TEXT("Failed to move file %s to %s!"), *From, *To);
    }
}

void FShaderCompileUtilities::DeleteFileHelper(const FString& Filename)
{
    // TODO: This logic came from FShaderCompileThreadRunnable::WriteNewTasks().
    // We can't avoid code duplication unless we refactored the local worker too.

    if (FPlatformFileManager::Get().GetPlatformFile().FileExists(*Filename))
    {
        bool bDeletedOutput = IFileManager::Get().Delete(*Filename, true, true);

        // Retry over the next two seconds if we couldn't delete it
        int32 RetryCount = 0;
        while (!bDeletedOutput && RetryCount < 200)
        {
            FPlatformProcess::Sleep(0.01f);
            bDeletedOutput = IFileManager::Get().Delete(*Filename, true, true);
            RetryCount++;
        }
        checkf(bDeletedOutput, TEXT("Failed to delete %s!"), *Filename);
    }
}

#endif // PLATFORM_WINDOWS