16.3K star! 微软开源AI Agent神器 OmniParser，让AI成为你的电脑操作专家

git clone https://github.com/microsoft/OmniParser.gitcd OmniParser

conda create -n "omni" python==3.12
conda activate omni

pip install --upgrade huggingface_hub
pip install gradio==4.14.0
pip install httpx==0.26.0
pip install httpcore==1.0.2
pip install anyio==4.2.0
pip install -r requirements.txt

import osfrom huggingface_hub import hf_hub_downloadfrom pathlib import Pathdef download_omniparser_models():"""下载OmniParser V2的模型文件"""try:
base_path = Path("weights")
base_path.mkdir(exist_ok=True)

files = ["icon_detect/train_args.yaml","icon_detect/model.pt","icon_detect/model.yaml","icon_caption/config.json","icon_caption/generation_config.json","icon_caption/model.safetensors"]print("开始下载模型文件...")for file in files:print(f"正在下载: {file}")
hf_hub_download(
repo_id="microsoft/OmniParser-v2.0",
filename=file,
local_dir=base_path
)

icon_caption_path = base_path / "icon_caption"icon_caption_florence_path = base_path / "icon_caption_florence"if icon_caption_path.exists():if icon_caption_florence_path.exists():import shutil
shutil.rmtree(icon_caption_florence_path)
icon_caption_path.rename(icon_caption_florence_path)print("\n所有文件下载完成！")except Exception as e:print(f"\n下载过程中出现错误: {str(e)}")print("请检查网络连接并重试")if __name__ == "__main__":
download_omniparser_models()

python gradio_demo.py

from fastapi import FastAPI, UploadFilefrom PIL import Imageimport ioimport uvicorn

app = FastAPI()@app.post("/analyze")async def analyze_screen(image: UploadFile):# 读取上传的图片image_data = await image.read()
image = Image.open(io.BytesIO(image_data))# 使用OmniParser处理图片# 这里添加OmniParser的处理逻辑return {"elements": [...]}# 返回识别到的元素信息if __name__ == "__main__":
uvicorn.run(app, host="0.0.0.0", port=8000)

import pyautoguiimport requestsfrom PIL import ImageGrabdef capture_screen():"""获取屏幕截图"""screenshot = ImageGrab.grab()return screenshotdef convert_coordinates(omni_coords):"""转换OmniParser坐标到PyAutoGUI坐标"""# 根据实际情况调整坐标转换逻辑return omni_coordsdef click_element(coords):"""执行点击操作"""pyautogui.click(coords[0], coords[1])def main():# 获取屏幕截图screenshot = capture_screen()# 发送到OmniParser服务files = {'image': ('screenshot.png', screenshot)}
response = requests.post('http://ubuntu-server:8000/analyze', files=files)# 处理返回结果elements = response.json()['elements']# 执行自动化操作for element in elements:
coords = convert_coordinates(element['coords'])
click_element(coords)if __name__ == "__main__":
main()