diff --git a/app.py b/app.py index 50c7285..9c6f935 100644 --- a/app.py +++ b/app.py @@ -401,9 +401,10 @@ with gr.Blocks(theme=gr.themes.Soft(font=[gr.themes.GoogleFont("Plus Jakarta San examples=[ [osp.join(example_portrait_dir, "s13.mp4")], # [osp.join(example_portrait_dir, "s18.mp4")], - [osp.join(example_portrait_dir, "s20.mp4")], + # [osp.join(example_portrait_dir, "s20.mp4")], [osp.join(example_portrait_dir, "s29.mp4")], [osp.join(example_portrait_dir, "s32.mp4")], + [osp.join(example_video_dir, "d3.mp4")], ], inputs=[retargeting_input_video], cache_examples=False, diff --git a/assets/docs/changelog/2024-08-19.md b/assets/docs/changelog/2024-08-19.md new file mode 100644 index 0000000..3f9a44d --- /dev/null +++ b/assets/docs/changelog/2024-08-19.md @@ -0,0 +1,49 @@ +# Image-driven animation and regional animation mode + +Inspired by [ComfyUI-AdvancedLivePortrait](https://github.com/PowerHouseMan/ComfyUI-AdvancedLivePortrait) ([@PowerHouseMan](https://github.com/PowerHouseMan)), we added the image-driven portrait animation and image-driven portrait video editing, as well as the regional animation mode. We also improved the absolute driving mode. + +## Arguments Description + +You can modify the argument options in `src/config/argument_config.py`. Below we explain the newly added arguments and the arguments that may confuse. + +### Image-driven portrait animation and portrait video editing ### + +- **driving**: You can set the image path of the driving option to your given driving image path as below: +```bash +python inference.py -d assets/examples/driving/d30.jpg +``` +You can also upload the driving image to the corresponding location in the Gradio interface. +

+ LivePortrait +
+ Image-driven portrait animation +

+ LivePortrait +
+ Image-driven portrait video editing +

+ +- **flag_relative_motion**: If the driving input is an image and you set `flag_relative_motion` to true, then the driving motion will be the motion deformation between the driving image and its canonical image. If you set `flag_relative_motion` to false, the driving motion will be the absolute motion of the driving image, which may result in greater expression driving strength and a certain degree of identity leakage. `flag_relative_motion` corresponds to the `relative motion` option in the Gradio interface. + +In addition, if both source input and driving input are images, the output will be an image. If source input is a video and driving input is an image, the animated result is a video, and the driving motion of each frame is provided by the driving image. In the Gradio interface, the output will be automatically saved in the corresponding format (image or video) and displayed in the corresponding window. + +### Regional animation mode ### +- **animation_region**: This argument contains five options. The `exp` option means that only the expression part of the driving input is used to drive the source input, `pose` means that only the head pose part of the driving input is used to drive the source input, `lip` means that only the lip movement part of the driving input is used to drive the source input, `eyes` means that only the eye movement part of the driving input is used to drive the source input, and `all` means that all motions of the driving input are used to drive the source input. Alternatively, you can also select the corresponding option in the following area in the gradio interface. + +

+ LivePortrait +
+ Regional animation mode +

+ +### Editing the lip region of the source video to the neutral expression ### +Some issues mentioned that in the `Retargeting Video` of the Gradio interface, it is hoped that not only the lips can be closed, but also the lip region of the source video can be edited into a neutral expression. Therefore, we provide a `keeping the lip silent` option. When you select this option, the lip region of the animated video will be the neutral expression. Since this uses a mode similar to absolute driving, this may cause inter-frame jitter or identity leakage in the animated video. Note that the neutral expression sometimes has a slightly open mouth. + +### More explanation ### +When both the source input and the driving input are videos, some issues mentioned that the output motion is a mixture of the source video and the driving video motions. This is because the `flag_relative_motion` argument is enabled by default. In other words, we are using relative driving, not absolute driving. The difference between the two is that `flag_relative_motion` means that the motion offset of the current driving frame relative to the first driving frame will be added to the motion of the source frame as the final driving motion, while `no_flag_relative_motion` means that the motion of the current driving frame will be directly used as the final driving motion. + +So if you want to keep only the motion of the driving video in the animated video, you can use: +```bash +python inference.py --no_flag_relative_motion +``` +Or you can uncheck the `relative motion` option in the Gradio interface. Using absolute driving may cause jitter or identity leakage in the animated video. diff --git a/assets/docs/image-driven-portrait-animation-2024-08-19.jpg b/assets/docs/image-driven-portrait-animation-2024-08-19.jpg new file mode 100644 index 0000000..56b4c56 Binary files /dev/null and b/assets/docs/image-driven-portrait-animation-2024-08-19.jpg differ diff --git a/assets/docs/image-driven-portrait-video-editing-2024-08-19.jpg b/assets/docs/image-driven-portrait-video-editing-2024-08-19.jpg new file mode 100644 index 0000000..54c4db7 Binary files /dev/null and b/assets/docs/image-driven-portrait-video-editing-2024-08-19.jpg differ diff --git a/assets/docs/regional-animation-2024-08-19.jpg b/assets/docs/regional-animation-2024-08-19.jpg new file mode 100644 index 0000000..ad9165e Binary files /dev/null and b/assets/docs/regional-animation-2024-08-19.jpg differ diff --git a/assets/examples/driving/d12.jpg b/assets/examples/driving/d12.jpg new file mode 100644 index 0000000..432186d Binary files /dev/null and b/assets/examples/driving/d12.jpg differ diff --git a/assets/examples/driving/d19.jpg b/assets/examples/driving/d19.jpg new file mode 100644 index 0000000..f5b380e Binary files /dev/null and b/assets/examples/driving/d19.jpg differ diff --git a/assets/examples/driving/d30.jpg b/assets/examples/driving/d30.jpg new file mode 100644 index 0000000..71ab7e0 Binary files /dev/null and b/assets/examples/driving/d30.jpg differ diff --git a/assets/examples/driving/d38.jpg b/assets/examples/driving/d38.jpg new file mode 100644 index 0000000..c797c73 Binary files /dev/null and b/assets/examples/driving/d38.jpg differ diff --git a/assets/examples/driving/d8.jpg b/assets/examples/driving/d8.jpg new file mode 100644 index 0000000..a7b1e8a Binary files /dev/null and b/assets/examples/driving/d8.jpg differ diff --git a/assets/examples/driving/d9.jpg b/assets/examples/driving/d9.jpg new file mode 100644 index 0000000..8b7844c Binary files /dev/null and b/assets/examples/driving/d9.jpg differ diff --git a/readme.md b/readme.md index e4a6488..f6df561 100644 --- a/readme.md +++ b/readme.md @@ -38,6 +38,7 @@ ## 🔥 Updates +- **`2024/08/19`**: 🖼️ We support **image driven and regional control**, insipred by [ComfyUI-AdvancedLivePortrait](https://github.com/PowerHouseMan/ComfyUI-AdvancedLivePortrait). See [**here**](./assets/docs/changelog/2024-08-19.md). - **`2024/08/06`**: 🎨 We support **precise portrait editing** in the Gradio interface, insipred by [ComfyUI-AdvancedLivePortrait](https://github.com/PowerHouseMan/ComfyUI-AdvancedLivePortrait). See [**here**](./assets/docs/changelog/2024-08-06.md). - **`2024/08/05`**: 📦 Windows users can now download the [one-click installer](https://huggingface.co/cleardusk/LivePortrait-Windows/blob/main/LivePortrait-Windows-v20240806.zip) for Humans mode and **Animals mode** now! For details, see [**here**](./assets/docs/changelog/2024-08-05.md). - **`2024/08/02`**: 😸 We released a version of the **Animals model**, along with several other updates and improvements. Check out the details [**here**](./assets/docs/changelog/2024-08-02.md)! diff --git a/src/utils/cropper.py b/src/utils/cropper.py index 10b4370..97e26c7 100644 --- a/src/utils/cropper.py +++ b/src/utils/cropper.py @@ -66,14 +66,14 @@ class Cropper(object): providers=face_analysis_wrapper_provider, ) self.face_analysis_wrapper.prepare(ctx_id=device_id, det_size=(512, 512), det_thresh=self.crop_cfg.det_thresh) - # self.face_analysis_wrapper.warmup() + self.face_analysis_wrapper.warmup() self.human_landmark_runner = HumanLandmark( ckpt_path=self.crop_cfg.landmark_ckpt_path, onnx_provider=device, device_id=device_id, ) - # self.human_landmark_runner.warmup() + self.human_landmark_runner.warmup() if self.image_type == "animal_face": from .animal_landmark_runner import XPoseRunner as AnimalLandmarkRunner @@ -83,7 +83,7 @@ class Cropper(object): embeddings_cache_path=self.crop_cfg.xpose_embedding_cache_path, flag_use_half_precision=kwargs.get("flag_use_half_precision", True), ) - # self.animal_landmark_runner.warmup() + self.animal_landmark_runner.warmup() def update_config(self, user_args): for k, v in user_args.items(): diff --git a/src/utils/resources/lip_array.pkl b/src/utils/resources/lip_array.pkl new file mode 100644 index 0000000..61fe803 Binary files /dev/null and b/src/utils/resources/lip_array.pkl differ