updates embedding examples with new embedding model
This commit is contained in:
Logan Kilpatrick
committed by
Ted Sanders
Ted Sanders
parent
7de3d50816
commit
fd181ec78f
@ -1,12 +1,13 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Code search\n",
|
||||
"\n",
|
||||
"We index our own openai-python code repository, and show how it can be searched. We implement a simple version of file parsing and extracting of functions from python files."
|
||||
"We index our own [openai-python code repository](https://github.com/openai/openai-python), and show how it can be searched. We implement a simple version of file parsing and extracting of functions from python files."
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -18,8 +19,8 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Total number of py files: 40\n",
|
||||
"Total number of functions extracted: 64\n"
|
||||
"Total number of py files: 51\n",
|
||||
"Total number of functions extracted: 97\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@ -63,18 +64,24 @@
|
||||
"\n",
|
||||
"# get user root directory\n",
|
||||
"root_dir = os.path.expanduser(\"~\")\n",
|
||||
"# note: for this code to work, the openai-python repo must be downloaded and placed in your root directory\n",
|
||||
"\n",
|
||||
"# path to code repository directory\n",
|
||||
"code_root = root_dir + \"/openai-python\"\n",
|
||||
"\n",
|
||||
"code_files = [y for x in os.walk(code_root) for y in glob(os.path.join(x[0], '*.py'))]\n",
|
||||
"print(\"Total number of py files:\", len(code_files))\n",
|
||||
"\n",
|
||||
"if len(code_files) == 0:\n",
|
||||
" print(\"Double check that you have downloaded the openai-python repo and set the code_root variable correctly.\")\n",
|
||||
"\n",
|
||||
"all_funcs = []\n",
|
||||
"for code_file in code_files:\n",
|
||||
" funcs = list(get_functions(code_file))\n",
|
||||
" for func in funcs:\n",
|
||||
" all_funcs.append(func)\n",
|
||||
"\n",
|
||||
"print(\"Total number of functions extracted:\", len(all_funcs))\n"
|
||||
"print(\"Total number of functions extracted:\", len(all_funcs))"
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -119,64 +126,57 @@
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>0</th>\n",
|
||||
" <td>def semantic_search(engine, query, documents):...</td>\n",
|
||||
" <td>semantic_search</td>\n",
|
||||
" <td>/examples/semanticsearch/semanticsearch.py</td>\n",
|
||||
" <td>[-0.038976121693849564, -0.0031428150832653046...</td>\n",
|
||||
" <td>def _console_log_level():\\n if openai.log i...</td>\n",
|
||||
" <td>_console_log_level</td>\n",
|
||||
" <td>/openai/util.py</td>\n",
|
||||
" <td>[0.03389773145318031, -0.004390408284962177, 0...</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1</th>\n",
|
||||
" <td>def main():\\n parser = argparse.ArgumentPar...</td>\n",
|
||||
" <td>main</td>\n",
|
||||
" <td>/examples/semanticsearch/semanticsearch.py</td>\n",
|
||||
" <td>[-0.024289356544613838, -0.017748363316059113,...</td>\n",
|
||||
" <td>def log_debug(message, **params):\\n msg = l...</td>\n",
|
||||
" <td>log_debug</td>\n",
|
||||
" <td>/openai/util.py</td>\n",
|
||||
" <td>[-0.004034275189042091, 0.004895383026450872, ...</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
" <td>def get_candidates(\\n prompt: str,\\n sto...</td>\n",
|
||||
" <td>get_candidates</td>\n",
|
||||
" <td>/examples/codex/backtranslation.py</td>\n",
|
||||
" <td>[-0.04161201789975166, -0.0169310811907053, 0....</td>\n",
|
||||
" <td>def log_info(message, **params):\\n msg = lo...</td>\n",
|
||||
" <td>log_info</td>\n",
|
||||
" <td>/openai/util.py</td>\n",
|
||||
" <td>[0.004882764536887407, 0.0033515947870910168, ...</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>3</th>\n",
|
||||
" <td>def rindex(lst: List, value: str) -> int:\\n ...</td>\n",
|
||||
" <td>rindex</td>\n",
|
||||
" <td>/examples/codex/backtranslation.py</td>\n",
|
||||
" <td>[-0.027255680412054062, -0.007931121625006199,...</td>\n",
|
||||
" <td>def log_warn(message, **params):\\n msg = lo...</td>\n",
|
||||
" <td>log_warn</td>\n",
|
||||
" <td>/openai/util.py</td>\n",
|
||||
" <td>[0.002535992069169879, -0.010829543694853783, ...</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>4</th>\n",
|
||||
" <td>def eval_candidate(\\n candidate_answer: str...</td>\n",
|
||||
" <td>eval_candidate</td>\n",
|
||||
" <td>/examples/codex/backtranslation.py</td>\n",
|
||||
" <td>[-0.00999179296195507, -0.01640152558684349, 0...</td>\n",
|
||||
" <td>def logfmt(props):\\n def fmt(key, val):\\n ...</td>\n",
|
||||
" <td>logfmt</td>\n",
|
||||
" <td>/openai/util.py</td>\n",
|
||||
" <td>[0.016732551157474518, 0.017367802560329437, 0...</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" code function_name \\\n",
|
||||
"0 def semantic_search(engine, query, documents):... semantic_search \n",
|
||||
"1 def main():\\n parser = argparse.ArgumentPar... main \n",
|
||||
"2 def get_candidates(\\n prompt: str,\\n sto... get_candidates \n",
|
||||
"3 def rindex(lst: List, value: str) -> int:\\n ... rindex \n",
|
||||
"4 def eval_candidate(\\n candidate_answer: str... eval_candidate \n",
|
||||
" code function_name \\\n",
|
||||
"0 def _console_log_level():\\n if openai.log i... _console_log_level \n",
|
||||
"1 def log_debug(message, **params):\\n msg = l... log_debug \n",
|
||||
"2 def log_info(message, **params):\\n msg = lo... log_info \n",
|
||||
"3 def log_warn(message, **params):\\n msg = lo... log_warn \n",
|
||||
"4 def logfmt(props):\\n def fmt(key, val):\\n ... logfmt \n",
|
||||
"\n",
|
||||
" filepath \\\n",
|
||||
"0 /examples/semanticsearch/semanticsearch.py \n",
|
||||
"1 /examples/semanticsearch/semanticsearch.py \n",
|
||||
"2 /examples/codex/backtranslation.py \n",
|
||||
"3 /examples/codex/backtranslation.py \n",
|
||||
"4 /examples/codex/backtranslation.py \n",
|
||||
"\n",
|
||||
" code_embedding \n",
|
||||
"0 [-0.038976121693849564, -0.0031428150832653046... \n",
|
||||
"1 [-0.024289356544613838, -0.017748363316059113,... \n",
|
||||
"2 [-0.04161201789975166, -0.0169310811907053, 0.... \n",
|
||||
"3 [-0.027255680412054062, -0.007931121625006199,... \n",
|
||||
"4 [-0.00999179296195507, -0.01640152558684349, 0... "
|
||||
" filepath code_embedding \n",
|
||||
"0 /openai/util.py [0.03389773145318031, -0.004390408284962177, 0... \n",
|
||||
"1 /openai/util.py [-0.004034275189042091, 0.004895383026450872, ... \n",
|
||||
"2 /openai/util.py [0.004882764536887407, 0.0033515947870910168, ... \n",
|
||||
"3 /openai/util.py [0.002535992069169879, -0.010829543694853783, ... \n",
|
||||
"4 /openai/util.py [0.016732551157474518, 0.017367802560329437, 0... "
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
@ -188,12 +188,109 @@
|
||||
"from openai.embeddings_utils import get_embedding\n",
|
||||
"\n",
|
||||
"df = pd.DataFrame(all_funcs)\n",
|
||||
"df['code_embedding'] = df['code'].apply(lambda x: get_embedding(x, engine='code-search-babbage-code-001'))\n",
|
||||
"df['code_embedding'] = df['code'].apply(lambda x: get_embedding(x, engine='text-embedding-ada-002'))\n",
|
||||
"df['filepath'] = df['filepath'].apply(lambda x: x.replace(code_root, \"\"))\n",
|
||||
"df.to_csv(\"output/code_search_openai-python.csv\", index=False)\n",
|
||||
"df.to_csv(\"data/code_search_openai-python.csv\", index=False)\n",
|
||||
"df.head()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/openai/tests/test_endpoints.py:test_completions score=0.826\n",
|
||||
"def test_completions():\n",
|
||||
" result = openai.Completion.create(prompt=\"This was a test\", n=5, engine=\"ada\")\n",
|
||||
" assert len(result.choices) == 5\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"----------------------------------------------------------------------\n",
|
||||
"/openai/tests/test_endpoints.py:test_completions_model score=0.811\n",
|
||||
"def test_completions_model():\n",
|
||||
" result = openai.Completion.create(prompt=\"This was a test\", n=5, model=\"ada\")\n",
|
||||
" assert len(result.choices) == 5\n",
|
||||
" assert result.model.startswith(\"ada\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"----------------------------------------------------------------------\n",
|
||||
"/openai/tests/test_endpoints.py:test_completions_multiple_prompts score=0.808\n",
|
||||
"def test_completions_multiple_prompts():\n",
|
||||
" result = openai.Completion.create(\n",
|
||||
" prompt=[\"This was a test\", \"This was another test\"], n=5, engine=\"ada\"\n",
|
||||
" )\n",
|
||||
" assert len(result.choices) == 10\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"----------------------------------------------------------------------\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from openai.embeddings_utils import cosine_similarity\n",
|
||||
"\n",
|
||||
"def search_functions(df, code_query, n=3, pprint=True, n_lines=7):\n",
|
||||
" embedding = get_embedding(code_query, engine='text-embedding-ada-002')\n",
|
||||
" df['similarities'] = df.code_embedding.apply(lambda x: cosine_similarity(x, embedding))\n",
|
||||
"\n",
|
||||
" res = df.sort_values('similarities', ascending=False).head(n)\n",
|
||||
" if pprint:\n",
|
||||
" for r in res.iterrows():\n",
|
||||
" print(r[1].filepath+\":\"+r[1].function_name + \" score=\" + str(round(r[1].similarities, 3)))\n",
|
||||
" print(\"\\n\".join(r[1].code.split(\"\\n\")[:n_lines]))\n",
|
||||
" print('-'*70)\n",
|
||||
" return res\n",
|
||||
"\n",
|
||||
"res = search_functions(df, 'Completions API tests', n=3)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/openai/validators.py:format_inferrer_validator score=0.751\n",
|
||||
"def format_inferrer_validator(df):\n",
|
||||
" \"\"\"\n",
|
||||
" This validator will infer the likely fine-tuning format of the data, and display it to the user if it is classification.\n",
|
||||
" It will also suggest to use ada and explain train/validation split benefits.\n",
|
||||
" \"\"\"\n",
|
||||
" ft_type = infer_task_type(df)\n",
|
||||
" immediate_msg = None\n",
|
||||
"----------------------------------------------------------------------\n",
|
||||
"/openai/validators.py:get_validators score=0.748\n",
|
||||
"def get_validators():\n",
|
||||
" return [\n",
|
||||
" num_examples_validator,\n",
|
||||
" lambda x: necessary_column_validator(x, \"prompt\"),\n",
|
||||
" lambda x: necessary_column_validator(x, \"completion\"),\n",
|
||||
" additional_column_validator,\n",
|
||||
" non_empty_field_validator,\n",
|
||||
"----------------------------------------------------------------------\n",
|
||||
"/openai/validators.py:infer_task_type score=0.738\n",
|
||||
"def infer_task_type(df):\n",
|
||||
" \"\"\"\n",
|
||||
" Infer the likely fine-tuning task type from the data\n",
|
||||
" \"\"\"\n",
|
||||
" CLASSIFICATION_THRESHOLD = 3 # min_average instances of each class\n",
|
||||
" if sum(df.prompt.str.len()) == 0:\n",
|
||||
" return \"open-ended generation\"\n",
|
||||
"----------------------------------------------------------------------\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"res = search_functions(df, 'fine-tuning input data validation logic', n=3)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
@ -203,48 +300,35 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/openai/tests/test_endpoints.py:test_completions_multiple_prompts score=0.681\n",
|
||||
"def test_completions_multiple_prompts():\n",
|
||||
" result = openai.Completion.create(\n",
|
||||
" prompt=[\"This was a test\", \"This was another test\"], n=5, engine=\"ada\"\n",
|
||||
" )\n",
|
||||
" assert len(result.choices) == 10\n",
|
||||
"\n",
|
||||
"/openai/validators.py:get_common_xfix score=0.793\n",
|
||||
"def get_common_xfix(series, xfix=\"suffix\"):\n",
|
||||
" \"\"\"\n",
|
||||
" Finds the longest common suffix or prefix of all the values in a series\n",
|
||||
" \"\"\"\n",
|
||||
" common_xfix = \"\"\n",
|
||||
" while True:\n",
|
||||
" common_xfixes = (\n",
|
||||
" series.str[-(len(common_xfix) + 1) :]\n",
|
||||
" if xfix == \"suffix\"\n",
|
||||
" else series.str[: len(common_xfix) + 1]\n",
|
||||
"----------------------------------------------------------------------\n",
|
||||
"/openai/tests/test_endpoints.py:test_completions score=0.675\n",
|
||||
"def test_completions():\n",
|
||||
" result = openai.Completion.create(prompt=\"This was a test\", n=5, engine=\"ada\")\n",
|
||||
" assert len(result.choices) == 5\n",
|
||||
"/openai/validators.py:common_completion_suffix_validator score=0.778\n",
|
||||
"def common_completion_suffix_validator(df):\n",
|
||||
" \"\"\"\n",
|
||||
" This validator will suggest to add a common suffix to the completion if one doesn't already exist in case of classification or conditional generation.\n",
|
||||
" \"\"\"\n",
|
||||
" error_msg = None\n",
|
||||
" immediate_msg = None\n",
|
||||
" optional_msg = None\n",
|
||||
" optional_fn = None\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"----------------------------------------------------------------------\n",
|
||||
"/openai/tests/test_api_requestor.py:test_requestor_sets_request_id score=0.635\n",
|
||||
"def test_requestor_sets_request_id(mocker: MockerFixture) -> None:\n",
|
||||
" # Fake out 'requests' and confirm that the X-Request-Id header is set.\n",
|
||||
"\n",
|
||||
" got_headers = {}\n",
|
||||
"\n",
|
||||
" def fake_request(self, *args, **kwargs):\n",
|
||||
" nonlocal got_headers\n",
|
||||
" ft_type = infer_task_type(df)\n",
|
||||
"----------------------------------------------------------------------\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from openai.embeddings_utils import cosine_similarity\n",
|
||||
"\n",
|
||||
"def search_functions(df, code_query, n=3, pprint=True, n_lines=7):\n",
|
||||
" embedding = get_embedding(code_query, engine='code-search-babbage-text-001')\n",
|
||||
" df['similarities'] = df.code_embedding.apply(lambda x: cosine_similarity(x, embedding))\n",
|
||||
"\n",
|
||||
" res = df.sort_values('similarities', ascending=False).head(n)\n",
|
||||
" if pprint:\n",
|
||||
" for r in res.iterrows():\n",
|
||||
" print(r[1].filepath+\":\"+r[1].function_name + \" score=\" + str(round(r[1].similarities, 3)))\n",
|
||||
" print(\"\\n\".join(r[1].code.split(\"\\n\")[:n_lines]))\n",
|
||||
" print('-'*70)\n",
|
||||
" return res\n",
|
||||
"res = search_functions(df, 'Completions API tests', n=3)\n"
|
||||
"res = search_functions(df, 'find common suffix', n=2, n_lines=10)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -256,90 +340,7 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/openai/validators.py:format_inferrer_validator score=0.655\n",
|
||||
"def format_inferrer_validator(df):\n",
|
||||
" \"\"\"\n",
|
||||
" This validator will infer the likely fine-tuning format of the data, and display it to the user if it is classification.\n",
|
||||
" It will also suggest to use ada and explain train/validation split benefits.\n",
|
||||
" \"\"\"\n",
|
||||
" ft_type = infer_task_type(df)\n",
|
||||
" immediate_msg = None\n",
|
||||
"----------------------------------------------------------------------\n",
|
||||
"/openai/validators.py:long_examples_validator score=0.649\n",
|
||||
"def long_examples_validator(df):\n",
|
||||
" \"\"\"\n",
|
||||
" This validator will suggest to the user to remove examples that are too long.\n",
|
||||
" \"\"\"\n",
|
||||
" immediate_msg = None\n",
|
||||
" optional_msg = None\n",
|
||||
" optional_fn = None\n",
|
||||
"----------------------------------------------------------------------\n",
|
||||
"/openai/validators.py:non_empty_completion_validator score=0.646\n",
|
||||
"def non_empty_completion_validator(df):\n",
|
||||
" \"\"\"\n",
|
||||
" This validator will ensure that no completion is empty.\n",
|
||||
" \"\"\"\n",
|
||||
" necessary_msg = None\n",
|
||||
" necessary_fn = None\n",
|
||||
" immediate_msg = None\n",
|
||||
"----------------------------------------------------------------------\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"res = search_functions(df, 'fine-tuning input data validation logic', n=3)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/openai/validators.py:common_completion_suffix_validator score=0.665\n",
|
||||
"def common_completion_suffix_validator(df):\n",
|
||||
" \"\"\"\n",
|
||||
" This validator will suggest to add a common suffix to the completion if one doesn't already exist in case of classification or conditional generation.\n",
|
||||
" \"\"\"\n",
|
||||
" error_msg = None\n",
|
||||
" immediate_msg = None\n",
|
||||
" optional_msg = None\n",
|
||||
" optional_fn = None\n",
|
||||
"\n",
|
||||
" ft_type = infer_task_type(df)\n",
|
||||
"----------------------------------------------------------------------\n",
|
||||
"/openai/validators.py:get_outfnames score=0.66\n",
|
||||
"def get_outfnames(fname, split):\n",
|
||||
" suffixes = [\"_train\", \"_valid\"] if split else [\"\"]\n",
|
||||
" i = 0\n",
|
||||
" while True:\n",
|
||||
" index_suffix = f\" ({i})\" if i > 0 else \"\"\n",
|
||||
" candidate_fnames = [\n",
|
||||
" fname.split(\".\")[0] + \"_prepared\" + suffix + index_suffix + \".jsonl\"\n",
|
||||
" for suffix in suffixes\n",
|
||||
" ]\n",
|
||||
" if not any(os.path.isfile(f) for f in candidate_fnames):\n",
|
||||
"----------------------------------------------------------------------\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"res = search_functions(df, 'find common suffix', n=2, n_lines=10)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/openai/cli.py:tools_register score=0.651\n",
|
||||
"/openai/cli.py:tools_register score=0.773\n",
|
||||
"def tools_register(parser):\n",
|
||||
" subparsers = parser.add_subparsers(\n",
|
||||
" title=\"Tools\", help=\"Convenience client side tools\"\n",
|
||||
@ -374,8 +375,9 @@
|
||||
"hash": "be4b5d5b73a21c599de40d6deb1129796d12dc1cc33a738f7bac13269cfcafe8"
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3.7.3 64-bit ('base': conda)",
|
||||
"name": "python3"
|
||||
"display_name": "openai-cookbook",
|
||||
"language": "python",
|
||||
"name": "openai-cookbook"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
@ -387,7 +389,7 @@
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.3"
|
||||
"version": "3.9.6"
|
||||
},
|
||||
"orig_nbformat": 4
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user