1. Request for additional logic to split into multiple batches to call and combine the results when GPT token is exceeded
2. Added EXCLUX_FILES
i'm using it very well. thank you
example (from gpt)
schema = {
'OPENAI_API_KEY': {'type': 'string', 'required': True},
'BITBUCKET_ACCESS_TOKEN': {'type': 'string', 'required': True},
'MODEL': {'type': 'string', 'required': True, 'allowed': ['gpt-4-turbo-preview', 'gpt-3.5-turbo-0125']},
'ORGANIZATION': {'type': 'string', 'required': False},
'MESSAGE': {'type': 'string', 'required': False},
'FILES_TO_REVIEW': {'type': 'string', 'required': False},
'EXCLUDE_FILES': {'type': 'string', 'required': False}, # 제외할 파일 리스트 추가
'CHATGPT_COMPLETION_FILEPATH': {'type': 'string', 'required': False},
'CHATGPT_CLIENT_FILEPATH': {'type': 'string', 'required': False},
'CHATGPT_PROMPT_MAX_TOKENS': {'type': 'integer', 'required': False, 'default': 0},
'DEBUG': {'type': 'boolean', 'required': False, 'default': False},
}
class ChatGPTCodereviewPipe(Pipe):
MAX_TOKENS = 4096 # 모델에 따라 최대 토큰 수 설정
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.auth_method_bitbucket = self.resolve_auth()
# Bitbucket
self.workspace = os.getenv('BITBUCKET_WORKSPACE')
self.repo_slug = os.getenv('BITBUCKET_REPO_SLUG')
self.bitbucket_client = BitbucketApiService(
self.auth_method_bitbucket, self.workspace, self.repo_slug)
# ChatGPT
self.open_api_key = self.get_variable('OPENAI_API_KEY')
self.organization = self.get_variable('ORGANIZATION')
self.model = self.get_variable('MODEL')
self.user_message_content = self.get_variable('MESSAGE')
self.files_to_review = self.get_variable("FILES_TO_REVIEW")
self.exclude_files = self.get_variable("EXCLUDE_FILES") # 제외할 파일 변수
self.completion_parameters_payload_file = self.get_variable('CHATGPT_COMPLETION_FILEPATH')
self.chatgpt_parameters_payload_file = self.get_variable('CHATGPT_CLIENT_FILEPATH')
self.chat_gpt_client = None
def get_diffs_to_review(self, pull_request_id):
diffs_text = self.bitbucket_client.get_pull_request_diffs(pull_request_id)
files_to_review = []
if self.files_to_review and self.files_to_review.split(','):
files_to_review = self.files_to_review.split(',')
exclude_files = []
if self.exclude_files and self.exclude_files.split(','):
exclude_files = self.exclude_files.split(',')
diffs = self.bitbucket_client.fetch_diffs(diffs_text, files_to_review, self.bitbucket_client.DIFF_DELIMITER)
# 제외할 파일 필터링
if exclude_files:
diffs = [diff for diff in diffs if not any(exclude_file in diff for exclude_file in exclude_files)]
return diffs
def split_messages_into_batches(self, messages, model, max_tokens):
"""메시지를 최대 토큰 수에 맞게 여러 배치로 분할"""
batches = []
current_batch = []
current_tokens = 0
for message in messages:
message_tokens = self.chat_gpt_client.num_tokens_from_messages([message], model)
if current_tokens + message_tokens > max_tokens:
batches.append(current_batch)
current_batch = []
current_tokens = 0
current_batch.append(message)
current_tokens += message_tokens
if current_batch:
batches.append(current_batch)
return batches
def get_suggestions(self, diffs_to_review):
messages = []
default_messages_system = {
"role": "system",
"content": DEFAULT_SYSTEM_PROMPT_FOR_CODE_REVIEW
}
messages.append(default_messages_system)
if self.user_message_content:
messages.append({"role": "system", "content": self.user_message_content})
default_messages_diffs = {
"role": "user",
"content": str(diffs_to_review)
}
messages.append(default_messages_diffs)
# 메시지를 여러 배치로 분할
message_batches = self.split_messages_into_batches(messages, self.model, self.MAX_TOKENS)
all_suggestions = {}
for batch in message_batches:
self.log_info(f"Processing ChatGPT batch with {len(batch)} messages...")
completion_params = {
'model': self.model,
'messages': batch,
}
# get payload with params for completion
if self.completion_parameters_payload_file:
users_completion_params = self.load_yaml(self.completion_parameters_payload_file)
self.log_info(f"ChatGPT configuration: completion parameters: {users_completion_params}")
completion_params.update(users_completion_params)
self.log_info(f"ChatGPT configuration: messages: {batch}")
self.log_info("Processing ChatGPT...")
start_time = time.time()
completion = None
try:
completion = self.chat_gpt_client.create_completion(**completion_params)
except BadRequestError as error:
self.fail(f"{str(error)}")
end_time = time.time()
self.log_debug(completion)
self.log_info(f"Processing ChatGPT takes: {round(end_time - start_time)} seconds")
self.log_info(f'ChatGPT completion tokens: {completion.usage}')
raw_suggestions = completion.choices[0].message.content
self.log_debug(raw_suggestions)
try:
suggestions = self.chat_gpt_client.fetch_json(raw_suggestions)
all_suggestions.update(suggestions)
except json.JSONDecodeError as error:
self.fail(str(error))
self.log_debug(all_suggestions)
return all_suggestions
@Oleksandr Kyrdan
