增加支持多模态,有图片识别功能和图片生成功能esperyong/gpt-cmd#2
![Logo of Sweep](/_next/image?url=%2Flogo.png&w=64&q=75)
增加支持多模态,有图片识别功能和图片生成功能
esperyong/gpt-cmd#2
> > >
✓ Completed in 8 minutes, 6 months ago using GPT-4 • Book a call • Report a bug
Progress
Create
gptcli/vision.py
8e032a4
1import base64
2import json
3
4import aiohttp
5
6
7class VisionAPIHandler:
8 def __init__(self, api_key: str):
9 self.api_key = api_key
10 self.headers = {"Authorization": f"Bearer {api_key}"}
11
12 async def generate_image(self, prompt: str, n_images: int = 1):
13 async with aiohttp.ClientSession() as session:
14 payload = {"prompt": prompt, "n": n_images}
15 async with session.post("https://api.openai.com/v1/images/generations", headers=self.headers, json=payload) as response:
16 if response.status == 200:
17 return await response.json()
18 else:
19 raise Exception(f"Failed to generate image: {response.status}")
20
21 async def recognize_image(self, image_path: str):
22 image_base64 = encode_image_to_base64(image_path)
23 async with aiohttp.ClientSession() as session:
24 payload = {"image": image_base64}
25 async with session.post("https://api.openai.com/v1/vision/recognitions", headers=self.headers, json=payload) as response:
26 if response.status == 200:
27 return await response.json()
28 else:
29 raise Exception(f"Failed to recognize image: {response.status}")
30
31def encode_image_to_base64(image_path: str) -> str:
32 with open(image_path, "rb") as image_file:
33 return base64.b64encode(image_file.read()).decode('utf-8')
34
35def decode_base64_to_image(base64_str: str, output_path: str):
36 with open(output_path, "wb") as output_file:
37 output_file.write(base64.b64decode(base64_str))
38
39def decode_response_to_text(response: dict) -> str:
40 return response.get("data", {}).get("text", "")
41
- This file will contain the classes and methods necessary for interacting with OpenAI's DALL-E 3 and GPT-4 vision APIs.
- Import necessary libraries for HTTP requests and asynchronous operations, considering the dependencies listed in pyproject.toml, such as
aiohttp
. - Define a class
VisionAPIHandler
with methodsgenerate_image
for interacting with DALL-E 3 andrecognize_image
for GPT-4 vision capabilities. These methods should accept parameters for API requests and return the API responses. - Include utility functions for encoding images to the required format for API requests and decoding API responses back into images or textual descriptions.
Run GitHub Actions for
gptcli/vision.py
Ran GitHub Actions for 8e032a4aca41545d846c369b4db52470f5a2547e:
Create
gptcli/cli_vision_commands.py
28bb96a
1import argparse
2import sys
3
4from gptcli.vision import VisionAPIHandler
5
6
7def cli_generate_image():
8 parser = argparse.ArgumentParser(description="Generate images from prompts.")
9 parser.add_argument("prompt", type=str, help="Prompt for generating images.")
10 parser.add_argument("--n_images", type=int, default=1, help="Number of images to generate.")
11 args = parser.parse_args()
12
13 try:
14 vision_handler = VisionAPIHandler(api_key="Your_API_Key_Here")
15 response = vision_handler.generate_image(prompt=args.prompt, n_images=args.n_images)
16 for image_data in response["data"]:
17 print(f"Generated Image URL: {image_data['url']}")
18 except Exception as e:
19 print(f"Error generating image: {e}")
20 sys.exit(1)
21
22def cli_recognize_image():
23 parser = argparse.ArgumentParser(description="Recognize text from an image.")
24 parser.add_argument("image_path", type=str, help="Path to the image file.")
25 args = parser.parse_args()
26
27 try:
28 vision_handler = VisionAPIHandler(api_key="Your_API_Key_Here")
29 response = vision_handler.recognize_image(image_path=args.image_path)
30 print(f"Recognized Text: {response['data']['text']}")
31 except Exception as e:
32 print(f"Error recognizing image: {e}")
33 sys.exit(1)
34
35if __name__ == "__main__":
36 parser = argparse.ArgumentParser(description="CLI for image recognition and generation.")
37 parser.add_argument("--generate", action="store_true", help="Generate images from prompts.")
38 parser.add_argument("--recognize", action="store_true", help="Recognize text from an image.")
39 args = parser.parse_args()
40
41 if args.generate:
42 cli_generate_image()
43 elif args.recognize:
44 cli_recognize_image()
45 else:
46 parser.print_help()
47
- This file will define the CLI commands for image recognition and generation.
- Import
VisionAPIHandler
fromgptcli/vision.py
and necessary CLI utilities. - Implement functions
cli_generate_image
andcli_recognize_image
that parse user input, call the respective methods inVisionAPIHandler
, and display the results to the user. - These functions should handle errors gracefully, providing user-friendly messages for common issues like invalid input or API errors.
Run GitHub Actions for
gptcli/cli_vision_commands.py
Ran GitHub Actions for 28bb96a1cec6bb4893ee66ebf293df9acde2e151:
ModifyChanged
gptcli/config.py:15-30
Changed gptcli/config.py
in c1dcaad
44 | return GptCliConfig( | 44 | return GptCliConfig( |
45 | **config, | 45 | **config, |
46 | ) | 46 | ) |
47 | dalle_api_key: Optional[str] = os.environ.get("DALLE_API_KEY") | ||
48 | gpt4_vision_api_key: Optional[str] = os.environ.get("GPT4_VISION_API_KEY") |
- Add new configuration options for DALL-E 3 and GPT-4 vision API keys,
dalle_api_key
andgpt4_vision_api_key
, respectively. - Ensure these new keys are optional to maintain backward compatibility and default to
None
if not set. - Update the
GptCliConfig
dataclass to include these new fields.
Modified file with Assistant API
Run GitHub Actions for
gptcli/config.py
Ran GitHub Actions for c1dcaadcd8d575dd5558739bfde8fc6f041dbf7b:
ModifyChanged
gptcli/openai.py:11-64
Changed gptcli/openai.py
in 8499c44
22 | kwargs["top_p"] = args["top_p"] | 22 | kwargs["top_p"] = args["top_p"] |
23 | 23 | ||
24 | if stream: | 24 | if stream: |
25 | from gptcli.vision import VisionAPIHandler | ||
26 | |||
27 | self.vision_handler = VisionAPIHandler(api_key=openai.api_key) | ||
28 | if input_type == "image": | ||
29 | if "image_path" in args: | ||
30 | try: | ||
31 | response = await self.vision_handler.recognize_image(image_path=args["image_path"]) | ||
32 | yield decode_response_to_text(response) | ||
33 | except Exception as e: | ||
34 | yield f"Error recognizing image: {e}" | ||
35 | elif "prompt" in args: | ||
36 | try: | ||
37 | response = await self.vision_handler.generate_image(prompt=args["prompt"], n_images=args.get("n_images", 1)) | ||
38 | for image_data in response["data"]: | ||
39 | yield f"Generated Image URL: {image_data['url']}" | ||
40 | except Exception as e: | ||
41 | yield f"Error generating image: {e}" | ||
42 | elif input_type == "text" and stream: | ||
25 | response_iter = self.client.chat.completions.create( | 43 | response_iter = self.client.chat.completions.create( |
26 | messages=cast(List[ChatCompletionMessageParam], messages), | 44 | messages=cast(List[ChatCompletionMessageParam], messages), |
27 | stream=True, | 45 | stream=True, |
... | |||
62 | 80 | ||
63 | def num_tokens_from_completion_openai(completion: Message, model: str) -> int: | 81 | def num_tokens_from_completion_openai(completion: Message, model: str) -> int: |
64 | return num_tokens_from_messages_openai([completion], model) | 82 | return num_tokens_from_messages_openai([completion], model) |
83 | from gptcli.vision import decode_response_to_text |
- Integrate calls to
VisionAPIHandler
within the existingOpenAICompletionProvider
class for cases where image processing is required. - This integration will likely involve checking the type of input (text vs. image) and deciding whether to call the chat completion API or the vision API based on this input.
- Add logic to handle the responses from the vision API, converting them into a format suitable for the CLI output.
Modified file with Assistant API
Run GitHub Actions for
gptcli/openai.py
Ran GitHub Actions for 8499c4483ebeb60a750fda739c1030cee78c5213:
ModifyChanged
gptcli/anthropic.py:33-67
Changed gptcli/anthropic.py
in c7c4216
66 | def num_tokens_from_completion_anthropic(message: Message, model: str) -> int: | 66 | def num_tokens_from_completion_anthropic(message: Message, model: str) -> int: |
67 | client = get_client() | 67 | client = get_client() |
68 | return client.count_tokens(message["content"]) | 68 | return client.count_tokens(message["content"]) |
69 | from gptcli.vision import VisionAPIHandler, decode_response_to_text | ||
70 | |||
71 | class AnthropicCompletionProvider(CompletionProvider): | ||
72 | def __init__(self): | ||
73 | self.vision_handler = VisionAPIHandler(api_key=api_key) | ||
74 | |||
75 | def complete( | ||
76 | self, messages: List[Message], args: dict, stream: bool = False | ||
77 | ) -> Iterator[str]: | ||
78 | input_type = args.get("input_type", "text") | ||
79 | if input_type == "image": | ||
80 | if "image_path" in args: | ||
81 | try: | ||
82 | response = await self.vision_handler.recognize_image(image_path=args["image_path"]) | ||
83 | yield decode_response_to_text(response) | ||
84 | except Exception as e: | ||
85 | yield f"Error recognizing image: {e}" | ||
86 | elif "prompt" in args: | ||
87 | try: | ||
88 | response = await self.vision_handler.generate_image(prompt=args["prompt"], n_images=args.get("n_images", 1)) | ||
89 | for image_data in response["data"]: | ||
90 | yield f"Generated Image URL: {image_data['url']}" | ||
91 | except Exception as e: | ||
92 | yield f"Error generating image: {e}" | ||
93 | elif input_type == "text": | ||
94 | kwargs = { | ||
95 | "prompt": make_prompt(messages), | ||
96 | "stop_sequences": [anthropic.HUMAN_PROMPT], | ||
97 | "max_tokens_to_sample": 4096, | ||
98 | "model": args["model"], | ||
99 | } | ||
100 | if "temperature" in args: | ||
101 | kwargs["temperature"] = args["temperature"] | ||
102 | if "top_p" in args: | ||
103 | kwargs["top_p"] = args["top_p"] | ||
104 | |||
105 | client = get_client() | ||
106 | if stream: | ||
107 | response = client.completions.create(**kwargs, stream=True) | ||
108 | else: | ||
109 | response = [client.completions.create(**kwargs, stream=False)] | ||
110 | |||
111 | for data in response: | ||
112 | next_completion = data.completion | ||
113 | yield next_completion |
- Similar to the modifications in
openai.py
, integrate calls toVisionAPIHandler
for image processing functionalities. - Ensure that the
AnthropicCompletionProvider
can handle both text and image inputs, directing each to the appropriate API (textual or vision) based on the input type.
Modified file with Assistant API
Run GitHub Actions for
gptcli/anthropic.py
Ran GitHub Actions for c7c4216a5083769d45b01f713af3dd188bd21409:
Plan
This is based on the results of the Planning step. The plan may expand from failed GitHub Actions runs.
Create
gptcli/vision.py
8e032a4
1import base64
2import json
3
4import aiohttp
5
6
7class VisionAPIHandler:
8 def __init__(self, api_key: str):
9 self.api_key = api_key
10 self.headers = {"Authorization": f"Bearer {api_key}"}
11
12 async def generate_image(self, prompt: str, n_images: int = 1):
13 async with aiohttp.ClientSession() as session:
14 payload = {"prompt": prompt, "n": n_images}
15 async with session.post("https://api.openai.com/v1/images/generations", headers=self.headers, json=payload) as response:
16 if response.status == 200:
17 return await response.json()
18 else:
19 raise Exception(f"Failed to generate image: {response.status}")
20
21 async def recognize_image(self, image_path: str):
22 image_base64 = encode_image_to_base64(image_path)
23 async with aiohttp.ClientSession() as session:
24 payload = {"image": image_base64}
25 async with session.post("https://api.openai.com/v1/vision/recognitions", headers=self.headers, json=payload) as response:
26 if response.status == 200:
27 return await response.json()
28 else:
29 raise Exception(f"Failed to recognize image: {response.status}")
30
31def encode_image_to_base64(image_path: str) -> str:
32 with open(image_path, "rb") as image_file:
33 return base64.b64encode(image_file.read()).decode('utf-8')
34
35def decode_base64_to_image(base64_str: str, output_path: str):
36 with open(output_path, "wb") as output_file:
37 output_file.write(base64.b64decode(base64_str))
38
39def decode_response_to_text(response: dict) -> str:
40 return response.get("data", {}).get("text", "")
41
Run GitHub Actions for
gptcli/vision.py
Create
gptcli/cli_vision_commands.py
28bb96a
1import argparse
2import sys
3
4from gptcli.vision import VisionAPIHandler
5
6
7def cli_generate_image():
8 parser = argparse.ArgumentParser(description="Generate images from prompts.")
9 parser.add_argument("prompt", type=str, help="Prompt for generating images.")
10 parser.add_argument("--n_images", type=int, default=1, help="Number of images to generate.")
11 args = parser.parse_args()
12
13 try:
14 vision_handler = VisionAPIHandler(api_key="Your_API_Key_Here")
15 response = vision_handler.generate_image(prompt=args.prompt, n_images=args.n_images)
16 for image_data in response["data"]:
17 print(f"Generated Image URL: {image_data['url']}")
18 except Exception as e:
19 print(f"Error generating image: {e}")
20 sys.exit(1)
21
22def cli_recognize_image():
23 parser = argparse.ArgumentParser(description="Recognize text from an image.")
24 parser.add_argument("image_path", type=str, help="Path to the image file.")
25 args = parser.parse_args()
26
27 try:
28 vision_handler = VisionAPIHandler(api_key="Your_API_Key_Here")
29 response = vision_handler.recognize_image(image_path=args.image_path)
30 print(f"Recognized Text: {response['data']['text']}")
31 except Exception as e:
32 print(f"Error recognizing image: {e}")
33 sys.exit(1)
34
35if __name__ == "__main__":
36 parser = argparse.ArgumentParser(description="CLI for image recognition and generation.")
37 parser.add_argument("--generate", action="store_true", help="Generate images from prompts.")
38 parser.add_argument("--recognize", action="store_true", help="Recognize text from an image.")
39 args = parser.parse_args()
40
41 if args.generate:
42 cli_generate_image()
43 elif args.recognize:
44 cli_recognize_image()
45 else:
46 parser.print_help()
47
Run GitHub Actions for
gptcli/cli_vision_commands.py
Modify
gptcli/config.py:15-30
Run GitHub Actions for
gptcli/config.py
Modify
gptcli/openai.py:11-64
Run GitHub Actions for
gptcli/openai.py
Run GitHub Actions for
gptcli/anthropic.py
Code Snippets Found
This is based on the results of the Searching step.
gptcli/openai.py:10-64
10
11class OpenAICompletionProvider(CompletionProvider):
12 def __init__(self):
13 self.client = OpenAI(api_key=openai.api_key)
14
15 def complete(
16 self, messages: List[Message], args: dict, stream: bool = False
17 ) -> Iterator[str]:
18 kwargs = {}
19 if "temperature" in args:
20 kwargs["temperature"] = args["temperature"]
21 if "top_p" in args:
22 kwargs["top_p"] = args["top_p"]
23
24 if stream:
25 response_iter = self.client.chat.completions.create(
26 messages=cast(List[ChatCompletionMessageParam], messages),
27 stream=True,
28 model=args["model"],
29 **kwargs,
30 )
31
32 for response in response_iter:
33 next_choice = response.choices[0]
34 if next_choice.finish_reason is None and next_choice.delta.content:
35 yield next_choice.delta.content
36 else:
37 response = self.client.chat.completions.create(
38 messages=cast(List[ChatCompletionMessageParam], messages),
39 model=args["model"],
40 stream=False,
41 **kwargs,
42 )
43 next_choice = response.choices[0]
44 if next_choice.message.content:
45 yield next_choice.message.content
46
47
48def num_tokens_from_messages_openai(messages: List[Message], model: str) -> int:
49 encoding = tiktoken.encoding_for_model(model)
50 num_tokens = 0
51 for message in messages:
52 # every message follows <im_start>{role/name}\n{content}<im_end>\n
53 num_tokens += 4
54 for key, value in message.items():
55 assert isinstance(value, str)
56 num_tokens += len(encoding.encode(value))
57 if key == "name": # if there's a name, the role is omitted
58 num_tokens += -1 # role is always required and always 1 token
59 num_tokens += 2 # every reply is primed with <im_start>assistant
60 return num_tokens
61
62
63def num_tokens_from_completion_openai(completion: Message, model: str) -> int:
64 return num_tokens_from_messages_openai([completion], model)
gptcli/anthropic.py:0-68
1import os
2from typing import Iterator, List
3import anthropic
4
5from gptcli.completion import CompletionProvider, Message
6
7api_key = os.environ.get("ANTHROPIC_API_KEY")
8
9
10def get_client():
11 if not api_key:
12 raise ValueError("ANTHROPIC_API_KEY environment variable not set")
13
14 return anthropic.Anthropic(api_key=api_key)
15
16
17def role_to_name(role: str) -> str:
18 if role == "system" or role == "user":
19 return anthropic.HUMAN_PROMPT
20 elif role == "assistant":
21 return anthropic.AI_PROMPT
22 else:
23 raise ValueError(f"Unknown role: {role}")
24
25
26def make_prompt(messages: List[Message]) -> str:
27 prompt = "\n".join(
28 [f"{role_to_name(message['role'])}{message['content']}" for message in messages]
29 )
30 prompt += f"{role_to_name('assistant')}"
31 return prompt
32
33
34class AnthropicCompletionProvider(CompletionProvider):
35 def complete(
36 self, messages: List[Message], args: dict, stream: bool = False
37 ) -> Iterator[str]:
38 kwargs = {
39 "prompt": make_prompt(messages),
40 "stop_sequences": [anthropic.HUMAN_PROMPT],
41 "max_tokens_to_sample": 4096,
42 "model": args["model"],
43 }
44 if "temperature" in args:
45 kwargs["temperature"] = args["temperature"]
46 if "top_p" in args:
47 kwargs["top_p"] = args["top_p"]
48
49 client = get_client()
50 if stream:
51 response = client.completions.create(**kwargs, stream=True)
52 else:
53 response = [client.completions.create(**kwargs, stream=False)]
54
55 for data in response:
56 next_completion = data.completion
57 yield next_completion
58
59
60def num_tokens_from_messages_anthropic(messages: List[Message], model: str) -> int:
61 prompt = make_prompt(messages)
62 client = get_client()
63 return client.count_tokens(prompt)
64
65
66def num_tokens_from_completion_anthropic(message: Message, model: str) -> int:
67 client = get_client()
68 return client.count_tokens(message["content"])
gptcli/config.py:0-45
1import os
2from typing import Dict, List, Optional, TypedDict
3from attr import dataclass
4import yaml
5
6from gptcli.assistant import AssistantConfig
7from gptcli.llama import LLaMAModelConfig
8
9
10CONFIG_FILE_PATHS = [
11 os.path.join(os.path.expanduser("~"), ".config", "gpt-cli", "gpt.yml"),
12 os.path.join(os.path.expanduser("~"), ".gptrc"),
13]
14
15
16@dataclass
17class GptCliConfig:
18 default_assistant: str = "general"
19 markdown: bool = True
20 show_price: bool = True
21 api_key: Optional[str] = os.environ.get("OPENAI_API_KEY")
22 openai_api_key: Optional[str] = os.environ.get("OPENAI_API_KEY")
23 anthropic_api_key: Optional[str] = os.environ.get("ANTHROPIC_API_KEY")
24 google_api_key: Optional[str] = os.environ.get("GOOGLE_API_KEY")
25 log_file: Optional[str] = None
26 log_level: str = "INFO"
27 conversations_save_directory: str = os.path.join(os.path.expanduser("~"), "Documents", "gpt-cli", "conversations")
28 conversations_render_directory: str = os.path.join(os.path.expanduser("~"), "develop", "opencsg", "work", "docs", "opencsg", "gpt-generate-md")
29 assistants: Dict[str, AssistantConfig] = {}
30 interactive: Optional[bool] = None
31 llama_models: Optional[Dict[str, LLaMAModelConfig]] = None
32
33
34def choose_config_file(paths: List[str]) -> str:
35 for path in paths:
36 if os.path.isfile(path):
37 return path
38 return ""
39
40
41def read_yaml_config(file_path: str) -> GptCliConfig:
42 with open(file_path, "r") as file:
43 config = yaml.safe_load(file)
44 return GptCliConfig(
45 **config,
pyproject.toml:0-48
1[project]
2name = "gpt-command-line"
3version = "0.1.4"
4description = "Command-line interface for ChatGPT, Claude and Bard"
5authors = [{name = "Val Kharitonov", email = "val@kharvd.com"}]
6readme = "README.md"
7license = {file = "LICENSE"}
8requires-python = ">=3.9"
9keywords = ["cli", "command-line", "assistant", "openai", "claude", "bard", "gpt-3", "gpt-4", "llm", "chatgpt", "gpt-cli", "google-bard", "anthropic", "gpt-client", "anthropic-claude", "palm2"]
10classifiers = [
11 "Development Status :: 4 - Beta",
12 "Environment :: Console",
13 "Intended Audience :: Developers",
14 "Intended Audience :: End Users/Desktop",
15 "Intended Audience :: Science/Research",
16 "License :: OSI Approved :: MIT License",
17 "Topic :: Scientific/Engineering :: Artificial Intelligence",
18]
19dependencies = [
20 "anthropic==0.7.7",
21 "attrs==23.1.0",
22 "black==23.1.0",
23 "google-generativeai==0.1.0",
24 "openai==1.3.8",
25 "prompt-toolkit==3.0.41",
26 "pytest==7.3.1",
27 "PyYAML==6.0",
28 "rich==13.7.0",
29 "tiktoken==0.5.2",
30 "tokenizers==0.15.0",
31 "typing_extensions==4.5.0",
32 "pytz==2023.3.post1",
33 "jinja2==3.1.2",
34]
35
36[project.optional-dependencies]
37llama = [
38 "llama-cpp-python==0.1.57",
39]
40
41[project.urls]
42"Homepage" = "https://github.com/kharvd/gpt-cli"
43
44[project.scripts]
45gpt = "gptcli.gpt:main"
46
47[build-system]
48requires = ["pip>=23.0.0", "setuptools>=58.0.0", "wheel"]