diff --git a/README.md b/README.md index 49e76c44..633a4ea8 100644 --- a/README.md +++ b/README.md @@ -527,6 +527,9 @@ If you need higher resolution, you could apply super resolution models such as [ - Jitter: There exists some jitter as the current pipeline adopts single-frame generation. +## Replicate +//TODO + # Citation ```bib @article{musetalk, @@ -542,3 +545,4 @@ If you need higher resolution, you could apply super resolution models such as [ 1. `other opensource model`: Other open-source models used must comply with their license, such as `whisper`, `ft-mse-vae`, `dwpose`, `S3FD`, etc.. 1. The testdata are collected from internet, which are available for non-commercial research purposes only. 1. `AIGC`: This project strives to impact the domain of AI-driven video generation positively. Users are granted the freedom to create videos using this tool, but they are expected to comply with local laws and utilize it responsibly. The developers do not assume any responsibility for potential misuse by users. + diff --git a/cog.yaml b/cog.yaml new file mode 100644 index 00000000..4f0ee988 --- /dev/null +++ b/cog.yaml @@ -0,0 +1,21 @@ +build: + gpu: true + cog_runtime: true + python_version: "3.10" + system_packages: + - libgl1-mesa-glx + - libglib2.0-0 + - ninja-build + - gcc + - g++ + - ffmpeg + + python_requirements: requirements.txt + run: + - pip install --no-cache-dir -U openmim + - mim install mmengine + - mim install "mmcv==2.0.1" + - mim install "mmdet==3.1.0" + - mim install "mmpose==1.1.0" + +predict: "predict.py:Predictor" diff --git a/requirements.txt b/requirements.txt index e87aa41d..c39f71e7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,20 +1,34 @@ -diffusers==0.30.2 -accelerate==0.28.0 +# PyTorch (CUDA 11.8 compatible) +torch==2.0.1+cu118 +torchvision==0.15.2+cu118 +torchaudio==2.0.2+cu118 +--extra-index-url https://download.pytorch.org/whl/cu118 + +# Core tools numpy==1.23.5 -tensorflow==2.12.0 -tensorboard==2.12.0 -opencv-python==4.9.0.80 +opencv-python-headless==4.9.0.80 +imageio[ffmpeg] +ffmpeg-python soundfile==0.12.1 +einops==0.8.1 +PyYAML +omegaconf +gdown +requests + +# Transformers and diffusion transformers==4.39.2 huggingface_hub==0.30.2 +diffusers==0.30.2 +accelerate==0.28.0 librosa==0.11.0 -einops==0.8.1 -gradio==5.24.0 -gdown -requests -imageio[ffmpeg] +# TensorFlow for face model (used in MuseTalk) +tensorflow==2.12.0 +tensorboard==2.12.0 -omegaconf -ffmpeg-python -moviepy +# Gradio for UI mode +gradio==5.24.0 + +# OpenMIM installer (used in cog.yaml to install mmcv/mmpose/etc.) +openmim==0.3.9 diff --git a/training-data.md b/training-data.md new file mode 100644 index 00000000..75dc95d1 --- /dev/null +++ b/training-data.md @@ -0,0 +1 @@ +### Training Data diff --git a/workflows/push.yaml b/workflows/push.yaml new file mode 100644 index 00000000..5d20efd6 --- /dev/null +++ b/workflows/push.yaml @@ -0,0 +1,58 @@ +name: Push to Replicate + +on: + # Workflow dispatch allows you to manually trigger the workflow from GitHub.com + # Go to your repo, click "Actions", click "Push to Replicate", click "Run workflow" + workflow_dispatch: + inputs: + model_name: + description: 'hunzai/musetalk' + + # # Uncomment these lines to trigger the workflow on every push to the main branch + # push: + # branches: + # - main + +jobs: + push_to_replicate: + name: Push to Replicate + + # If your model is large, the default GitHub Actions runner may not + # have enough disk space. If you need more space you can set up a + # bigger runner on GitHub. + runs-on: ubuntu-latest + + steps: + # This action cleans up disk space to make more room for your + # model code, weights, etc. + - name: Free disk space + uses: jlumbroso/free-disk-space@v1.3.1 + with: + tool-cache: false + docker-images: false + + - name: Checkout + uses: actions/checkout@v4 + + # This action installs Docker buildx and Cog (and optionally CUDA) + - name: Setup Cog + uses: replicate/setup-cog@v2 + with: + # If you add a CI auth token to your GitHub repository secrets, + # the action will authenticate with Replicate automatically so you + # can push your model without needing to pass in a token. + # + # To genereate a CLI auth token, run `cog login` or visit this page + # in your browser: https://replicate.com/account/api-token + token: ${{ secrets.REPLICATE_CLI_AUTH_TOKEN }} + + # If you trigger the workflow manually, you can specify the model name. + # If you leave it blank (or if the workflow is triggered by a push), the + # model name will be derived from the `image` value in cog.yaml. + - name: Push to Replicate + run: | + if [ -n "${{ inputs.model_name }}" ]; then + cog push r8.im/${{ inputs.model_name }} + else + cog push + fi