UE外包团队：基于Metahuman开发虚拟数字人
下面本文将分享一种使用 UE4 来渲染带有相机参数的多视点图片的方法。在正式开始之前，首先来介绍一下 MetaHuman，它就是我们要渲染的对象。在计算机视觉、三维重建领域，人脸方向的研究一直是一个热点，而 MetaHuman 则是一个高保真的数字人开源框架，我们可以利用 MetaHuman 来设计出自己想要的数字人形象，然后渲染出其多视点图片，用作进一步的研究。
步骤一：角色导入以及人脸定位
由于我只想渲染 MetaHuman 的人脸区域，因此我需要将其余身体部分移动到相机视野之外。需要移动的身体部位有：Torso、Legs、Feet。
接着进行 MetaHuman 的定位，为了方便，我们可以将其 Face 位置定位在坐标原点（0,0,0），这是为了方便后面指定相机的位置与旋转。
步骤二：指定相机运动方式：位置和旋转
在这一步，我希望相机能够以 MetaHuman 的人脸（也就是坐标原点）为中心，做球形旋转运动。关于相机的外参，做如下设置：
- 相机到坐标原点的距离（即：球半径）为35
- 相机的俯仰角设置为如下角度：[-30， -15， 0， 15， 30]
- 在每个俯仰角下，偏航角区间设置为：[-180, 180]， 每间隔角度 10 定义一个关键点（即：拍摄一张照片）
渲染的相关的设置如下：
- 相机的视场角设置为：100
- 所渲染图片的分辨率：512
- 帧与帧之间的延迟：1.5
相机内参的设置见：步骤三具体的渲染代码。
import sys import unreal, os, json, math # sequence asset path sequence_asset_path = '/Game/Render_Sequence.Render_Sequence' # read json file def read_json_file(path): file = open(path, "rb") file_json = json.load(file) export_path = file_json['output_image_path'] # image export path film_fov = file_json.get('film_fov', None) # camera film fov film_resolution = file_json.get("film_resolution", 512) # camera file resolution delay_every_frame = file_json.get("delay_every_frame", 3.0) # sequence run delay every frame camera_json = file_json['camera_data'] # camera data camera_transform_array = [] camera_rotation_array = [] camera_name_array = [] for camera in camera_json: camera_transform = camera_json[camera]["camera_transform"] for index in range(len(camera_transform)): camera_transform_array.append(unreal.Transform( location=[camera_transform[index][0], camera_transform[index][1], camera_transform[index][2]], rotation=[camera_transform[index][3], camera_transform[index][4], camera_transform[index][5]], scale=[1.0, 1.0, 1.0] )) camera_rotation_array.append(unreal.Rotator( roll=camera_transform[index][3], pitch=camera_transform[index][4], yaw=camera_transform[index][5] )) camera_name_array.append(camera) return camera_transform_array, camera_rotation_array, camera_name_array, export_path, film_fov, film_resolution, delay_every_frame def create_sequence(asset_name, camera_transform_array, camera_rotation_array, camera_name_array, film_fov, package_path='/Game/'): # create a sequence sequence = unreal.AssetToolsHelpers.get_asset_tools().create_asset(asset_name, package_path, unreal.LevelSequence, unreal.LevelSequenceFactoryNew()) sequence.set_display_rate(unreal.FrameRate(numerator=1, denominator=1)) sequence.set_playback_start_seconds(0) sequence.set_playback_end_seconds(len(camera_transform_array)) # add a camera cut track camera_cut_track = sequence.add_master_track(unreal.MovieSceneCameraCutTrack) print(camera_transform_array[0], '=======================================================================') print(camera_rotation_array[0].get_editor_property("roll")) # 添加关键帧 for index in range(0, len(camera_name_array)): camera_cut_section = camera_cut_track.add_section() camera_cut_section.set_start_frame_bounded(6 * index) camera_cut_section.set_end_frame_bounded(6 * index + len(camera_transform_array)) camera_cut_section.set_start_frame_seconds(6 * index) camera_cut_section.set_end_frame_seconds(6 * index + len(camera_transform_array)) # Create a cine camera actor # 在世界编辑器中创建一个电影摄像机CineCameraActor,并指定位置和旋转 camera_actor = unreal.EditorLevelLibrary().spawn_actor_from_class(unreal.CineCameraActor, unreal.Vector(0, 0, 0), unreal.Rotator(0, 0, 0)) # 指定电影摄像机CineCameraActor的名称 camera_actor.set_actor_label(camera_name_array[index]) # 返回CineCamera组件 camera_component = camera_actor.get_cine_camera_component() ratio = math.tan(film_fov / 360.0 * math.pi) * 2 # step1：胶片背板设置 filmback = camera_component.get_editor_property("filmback") # sensor_height：胶片或数字传感器的垂直尺寸，以毫米为单位 filmback.set_editor_property("sensor_height", 60) # sensor_width：胶片或数字传感器的水平尺寸，以毫米为单位 filmback.set_editor_property("sensor_width", 60) # step2：镜头设置 lens_settings = camera_component.get_editor_property("lens_settings") lens_settings.set_editor_property("min_focal_length", 4.0) lens_settings.set_editor_property("max_focal_length", 1000) lens_settings.set_editor_property("min_f_stop", 1.2) lens_settings.set_editor_property("max_f_stop", 500) print(lens_settings) # step3：聚焦设置 focal_length = 45 # 焦距 current_aperture = 500 # 光圈 camera_component.set_editor_property("current_focal_length", focal_length) # 设置当前焦距 camera_component.set_editor_property("current_aperture", current_aperture) # 设置当前光圈 fov = camera_component.get_editor_property("field_of_view") print(camera_component.get_editor_property("current_focal_length")) print(camera_component.get_editor_property("current_aperture")) print(f"fov - {fov}") # add a binding for the camera camera_binding = sequence.add_possessable(camera_actor) transform_track = camera_binding.add_track(unreal.MovieScene3DTransformTrack) transform_section = transform_track.add_section() transform_section.set_start_frame_bounded(6 * index) transform_section.set_end_frame_bounded(6 * index + len(camera_transform_array)) transform_section.set_start_frame_seconds(6 * index) transform_section.set_end_frame_seconds(6 * index + len(camera_transform_array)) # get channel for location_x location_y location_z channel_location_x = transform_section.get_channels()[0] channel_location_y = transform_section.get_channels()[1] channel_location_z = transform_section.get_channels()[2] # get key for rotation_y rotation_z channel_rotation_x = transform_section.get_channels()[3] channel_rotation_y = transform_section.get_channels()[4] channel_rotation_z = transform_section.get_channels()[5] # camera_transform = camera_transform_array[index] # camera_location_x = camera_transform.get_editor_property("translation").get_editor_property("x") # camera_location_y = camera_transform.get_editor_property("translation").get_editor_property("y") # camera_location_z = camera_transform.get_editor_property("translation").get_editor_property("z") for i in range(len(camera_transform_array)): camera_transform = camera_transform_array[i] camera_location_x = camera_transform.get_editor_property("translation").get_editor_property("x") camera_location_y = camera_transform.get_editor_property("translation").get_editor_property("y") camera_location_z = camera_transform.get_editor_property("translation").get_editor_property("z") camera_rotation = camera_rotation_array[i] camera_rotate_roll = camera_rotation.get_editor_property("roll") camera_rotate_pitch = camera_rotation.get_editor_property("pitch") camera_rotate_yaw = camera_rotation.get_editor_property("yaw") new_time = unreal.FrameNumber(value=i) channel_location_x.add_key(new_time, camera_location_x, 0.0) channel_location_y.add_key(new_time, camera_location_y, 0.0) channel_location_z.add_key(new_time, camera_location_z, 0.0) channel_rotation_x.add_key(new_time, camera_rotate_roll, 0.0) channel_rotation_y.add_key(new_time, camera_rotate_pitch, 0.0) channel_rotation_z.add_key(new_time, camera_rotate_yaw, 0.0) # if i % 5 == 0: # print(new_time, [camera_location_x, camera_location_y, camera_location_z, camera_rotate_roll, camera_rotate_pitch, camera_rotate_yaw]) # add the binding for the camera cut section camera_binding_id = sequence.make_binding_id(camera_binding, unreal.MovieSceneObjectBindingSpace.LOCAL) camera_cut_section.set_camera_binding_id(camera_binding_id) # save sequence asset unreal.EditorAssetLibrary.save_loaded_asset(sequence, False) return sequence # sequence movie def render_sequence_to_movie(export_path, film_resolution, delay_every_frame, on_finished_callback): # 1) Create an instance of our UAutomatedLevelSequenceCapture and override all of the settings on it. This class is currently # set as a config class so settings will leak between the Unreal Sequencer Render-to-Movie UI and this object. To work around # this, we set every setting via the script so that no changes the user has made via the UI will affect the script version. # The users UI settings will be reset as an unfortunate side effect of this. capture_settings = unreal.AutomatedLevelSequenceCapture() # Set all POD settings on the UMovieSceneCapture # capture_settings.settings.output_directory = unreal.DirectoryPath("../../../unreal-render-release/Saved/VideoCaptures1/") capture_settings.settings.output_directory = unreal.DirectoryPath(export_path) # If you game mode is implemented in Blueprint, load_asset(...) is going to return you the C++ type ('Blueprint') and not what the BP says it inherits from. # Instead, because game_mode_override is a TSubclassOf<AGameModeBase> we can use unreal.load_class to get the UClass which is implicitly convertable. # ie: capture_settings.settings.game_mode_override = unreal.load_class(None, "/Game/AI/TestingSupport/AITestingGameMode.AITestingGameMode_C") capture_settings.settings.game_mode_override = None capture_settings.settings.output_format = "{camera}" + "_{frame}" capture_settings.settings.overwrite_existing = False capture_settings.settings.use_relative_frame_numbers = False capture_settings.settings.handle_frames = 0 capture_settings.settings.zero_pad_frame_numbers = 4 # If you wish to override the output framerate you can use these two lines, otherwise the framerate will be derived from the sequence being rendered capture_settings.settings.use_custom_frame_rate = False # capture_settings.settings.custom_frame_rate = unreal.FrameRate(24,1) capture_settings.settings.resolution.res_x = film_resolution capture_settings.settings.resolution.res_y = film_resolution capture_settings.settings.enable_texture_streaming = False capture_settings.settings.cinematic_engine_scalability = True capture_settings.settings.cinematic_mode = True capture_settings.settings.allow_movement = False # Requires cinematic_mode = True capture_settings.settings.allow_turning = False # Requires cinematic_mode = True capture_settings.settings.show_player = False # Requires cinematic_mode = True capture_settings.settings.show_hud = False # Requires cinematic_mode = True capture_settings.use_separate_process = False capture_settings.close_editor_when_capture_starts = False # Requires use_separate_process = True capture_settings.additional_command_line_arguments = "-NOSCREENMESSAGES" # Requires use_separate_process = True capture_settings.inherited_command_line_arguments = "" # Requires use_separate_process = True # Set all the POD settings on UAutomatedLevelSequenceCapture capture_settings.use_custom_start_frame = False # If False, the system will automatically calculate the start based on sequence content capture_settings.use_custom_end_frame = False # If False, the system will automatically calculate the end based on sequence content capture_settings.custom_start_frame = unreal.FrameNumber(0) # Requires use_custom_start_frame = True capture_settings.custom_end_frame = unreal.FrameNumber(0) # Requires use_custom_end_frame = True capture_settings.warm_up_frame_count = 0 capture_settings.delay_before_warm_up = 0 capture_settings.delay_before_shot_warm_up = 0 capture_settings.delay_every_frame = delay_every_frame capture_settings.write_edit_decision_list = True # Tell the capture settings which level sequence to render with these settings. The asset does not need to be loaded, # as we're only capturing the path to it and when the PIE instance is created it will load the specified asset. # If you only had a reference to the level sequence, you could use "unreal.SoftObjectPath(mysequence.get_path_name())" capture_settings.level_sequence_asset = unreal.SoftObjectPath(sequence_asset_path) # To configure the video output we need to tell the capture settings which capture protocol to use. The various supported # capture protocols can be found by setting the Unreal Content Browser to "Engine C++ Classes" and filtering for "Protocol" # ie: CompositionGraphCaptureProtocol, ImageSequenceProtocol_PNG, etc. Do note that some of the listed protocols are not intended # to be used directly. # Right click on a Protocol and use "Copy Reference" and then remove the extra formatting around it. ie: # Class'/Script/MovieSceneCapture.ImageSequenceProtocol_PNG' gets transformed into "/Script/MovieSceneCapture.ImageSequenceProtocol_PNG" capture_settings.set_image_capture_protocol_type( unreal.load_class(None, "/Script/MovieSceneCapture.ImageSequenceProtocol_PNG")) # After we have set the capture protocol to a soft class path we can start editing the settings for the instance of the protocol that is internallyc reated. capture_settings.get_image_capture_protocol().compression_quality = 85 # Finally invoke Sequencer's Render to Movie functionality. This will examine the specified settings object and either construct a new PIE instance to render in, # or create and launch a new process (optionally shutting down your editor). unreal.SequencerTools.render_movie(capture_settings, on_finished_callback) # sequence end call back def on_render_movie_finished(success): unreal.log('on_render_movie_finished is called') if success: unreal.log('---end LevelSequenceTask') # check asset is exist def check_sequence_asset_exist(root_dir): sequence_path = os.path.join(root_dir, 'Content', 'Render_Sequence.uasset') if (os.path.exists(sequence_path)): unreal.log('---sequence is exist os path---') os.remove(sequence_path) else: unreal.log('---sequence is not exist os path---') if unreal.EditorAssetLibrary.does_asset_exist(sequence_asset_path): unreal.log('---sequence dose asset exist---') unreal.EditorAssetLibrary.delete_asset(sequence_asset_path) else: unreal.log('---sequence does not asset exist---') sequence_asset_data = unreal.AssetRegistryHelpers.get_asset_registry().get_asset_by_object_path(sequence_asset_path) if sequence_asset_data: unreal.log('---sequence is exist on content---') sequence_asset = unreal.AssetData.get_asset(sequence_asset_data) unreal.EditorAssetLibrary.delete_loaded_asset(sequence_asset) else: unreal.log('---sequence is not exit on content---') def main(): # bind event on_finished_callback = unreal.OnRenderMovieStopped() on_finished_callback.bind_callable(on_render_movie_finished) # get json file root_dir = unreal.SystemLibrary.get_project_directory() json_path = os.path.join(root_dir, 'RawData', 'camera_data.json') # get json data camera_transform_array, camera_rotation_array, camera_name_array, export_path, film_fov, film_resolution, delay_every_frame = read_json_file( json_path) # check sequence is asset check_sequence_asset_exist(root_dir) # create sequence create_sequence('Render_Sequence', camera_transform_array, camera_rotation_array, camera_name_array, film_fov) # run sequence render_sequence_to_movie(export_path, film_resolution, delay_every_frame, on_finished_callback)
REF：https://blog.csdn.net/javastart/article/details/129017627
有UE4或UE5相关需求欢迎联系本团队
QQ：372900288 VX：Liuxiang0884
