summaryrefslogtreecommitdiff
path: root/generators
diff options
context:
space:
mode:
Diffstat (limited to 'generators')
-rw-r--r--generators/datageneration.ipynb317
1 files changed, 252 insertions, 65 deletions
diff --git a/generators/datageneration.ipynb b/generators/datageneration.ipynb
index ee65ead..85c6e02 100644
--- a/generators/datageneration.ipynb
+++ b/generators/datageneration.ipynb
@@ -41,13 +41,18 @@
},
{
"cell_type": "code",
- "execution_count": 3,
- "id": "c75240e0-8392-4a7b-9999-dc528b3d17a1",
+ "execution_count": 60,
+ "id": "27488c82-1e9b-4873-9e6b-e0b5937fc51d",
"metadata": {},
"outputs": [],
"source": [
"from collections import defaultdict\n",
- "\n",
+ "lexnames = set()\n",
+ "lexname_blacklist = {\n",
+ " 'noun.plant',\n",
+ " 'noun.animal',\n",
+ " 'noun.person'\n",
+ "}\n",
"def groups_for_pos(pos='as'):\n",
" dsynonyms = []\n",
" for word in wn.words():\n",
@@ -55,11 +60,12 @@
" synonymss = wn.synonyms(word)\n",
" syns = set()\n",
" for synset, synonyms in zip(synsets, synonymss):\n",
- " if synset.pos() in pos: # 'as'\n",
+ " if synset.lexname() in lexname_blacklist:\n",
+ " continue\n",
+ " if synset.pos() in pos:\n",
" syns |= set(synonyms)\n",
" if len(syns) >= 4:\n",
" clues = [format(clue) for clue in syns]\n",
- " \n",
" clues.append(format(word))\n",
" dsynonyms.append(dict(answer=word, hint=f'synonyms for {format(word)}', clues=clues))\n",
" \"\"\" \n",
@@ -71,8 +77,16 @@
" clues.append(format(word))\n",
" dsynonyms[word] = dict(group=f'synonyms for {format(word)}', clues=clues)\n",
" \"\"\"\n",
- " return dsynonyms\n",
- "\n",
+ " return dsynonyms"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 61,
+ "id": "445789e7-3808-4a28-9df5-9a69313cb4c2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
"dadj = groups_for_pos('as')\n",
"dverb = groups_for_pos('v')\n",
"dnoun = groups_for_pos('n')\n",
@@ -81,7 +95,7 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 30,
"id": "7e552fc8-03b2-4b8f-b6f6-072d580702bc",
"metadata": {},
"outputs": [
@@ -91,7 +105,7 @@
"(3976, 7141, 19563, 490)"
]
},
- "execution_count": 4,
+ "execution_count": 30,
"metadata": {},
"output_type": "execute_result"
}
@@ -102,20 +116,33 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 62,
"id": "8d780210-38a4-4f8d-ae2e-b4631cb06368",
"metadata": {},
- "outputs": [],
- "source": []
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "(3976, 7141, 13824, 490)"
+ ]
+ },
+ "execution_count": 62,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(dadj), len(dverb), len(dnoun), len(dadverb)"
+ ]
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 33,
"id": "48233554-2634-4a5e-9013-4e45c6f7d3d9",
"metadata": {},
"outputs": [],
"source": [
- "# flag button for reporting"
+ "# flag button for reporting, definitions in webcollectionscollections"
]
},
{
@@ -129,7 +156,115 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "ecc5527c-a0b0-4e48-ae0a-4cb3a1a8a12b",
+ "id": "f4eb0f95-901d-43d8-8ba7-6103b3a0f6be",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 57,
+ "id": "bdcb3a19-a5cf-48e9-ab6f-aaeed50e2c31",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'adj.all',\n",
+ " 'adj.pert',\n",
+ " 'adj.ppl',\n",
+ " 'adv.all',\n",
+ " 'noun.Tops',\n",
+ " 'noun.act',\n",
+ " 'noun.animal',\n",
+ " 'noun.artifact',\n",
+ " 'noun.attribute',\n",
+ " 'noun.body',\n",
+ " 'noun.cognition',\n",
+ " 'noun.communication',\n",
+ " 'noun.event',\n",
+ " 'noun.feeling',\n",
+ " 'noun.food',\n",
+ " 'noun.group',\n",
+ " 'noun.location',\n",
+ " 'noun.motive',\n",
+ " 'noun.object',\n",
+ " 'noun.person',\n",
+ " 'noun.phenomenon',\n",
+ " 'noun.plant',\n",
+ " 'noun.possession',\n",
+ " 'noun.process',\n",
+ " 'noun.quantity',\n",
+ " 'noun.relation',\n",
+ " 'noun.shape',\n",
+ " 'noun.state',\n",
+ " 'noun.substance',\n",
+ " 'noun.time',\n",
+ " 'verb.body',\n",
+ " 'verb.change',\n",
+ " 'verb.cognition',\n",
+ " 'verb.communication',\n",
+ " 'verb.competition',\n",
+ " 'verb.consumption',\n",
+ " 'verb.contact',\n",
+ " 'verb.creation',\n",
+ " 'verb.emotion',\n",
+ " 'verb.motion',\n",
+ " 'verb.perception',\n",
+ " 'verb.possession',\n",
+ " 'verb.social',\n",
+ " 'verb.stative',\n",
+ " 'verb.weather'}"
+ ]
+ },
+ "execution_count": 57,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 50,
+ "id": "d3a0f1e9-99e5-4aa2-bd72-d1d200bf2b40",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "'noun.plant'"
+ ]
+ },
+ "execution_count": 50,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "syn.lexname()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "64ad6047-a09b-41c3-85b6-65476d8dba0e",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "88a4aeab-8123-435e-9b44-3e568102f0b1",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b6459fdf-fe4e-4dcf-81ef-1688de01be95",
"metadata": {},
"outputs": [],
"source": []
@@ -194,26 +329,24 @@
},
{
"cell_type": "code",
- "execution_count": 11,
+ "execution_count": 10,
"id": "def43999-d789-4e5c-bb27-4fd29074c875",
"metadata": {},
- "outputs": [
- {
- "ename": "IndentationError",
- "evalue": "unindent does not match any outer indentation level (<tokenize>, line 84)",
- "output_type": "error",
- "traceback": [
- "\u001b[0;36m File \u001b[0;32m<tokenize>:84\u001b[0;36m\u001b[0m\n\u001b[0;31m r = ratio(a, b)\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mIndentationError\u001b[0m\u001b[0;31m:\u001b[0m unindent does not match any outer indentation level\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"from Levenshtein import ratio\n",
"def similar(a, b):\n",
" a, b = a.lower(), b.lower()\n",
" if len(a) > len(b):\n",
" a, b = b, a\n",
- " if a + 's' == b or a + 'es' == b or a + 'ing' == b or a + 'ly' == b\n",
+ " if a + 's' == b or a + 'es' == b or a + 'ing' == b or a + 'ly' == b:\n",
+ " return True\n",
+ " r = ratio(a, b)\n",
+ " if .8 <= r <= .9:\n",
+ " pass\n",
+ " #print(a, b, r)\n",
+ " return r >= .85\n",
+ "\"\"\"\n",
"Skip to Main\n",
"datageneration\n",
"Last Checkpoint: 11 days ago\n",
@@ -250,7 +383,7 @@
" \n",
" clues.append(format(word))\n",
" dsynonyms.append(dict(answer=word, hint=f'synonyms for {format(word)}', clues=clues))\n",
- " \"\"\" \n",
+ " \n",
" ok = True\n",
" for clue in clues:\n",
" if clue in dsynonyms:\n",
@@ -258,7 +391,7 @@
" if ok:\n",
" clues.append(format(word))\n",
" dsynonyms[word] = dict(group=f'synonyms for {format(word)}', clues=clues)\n",
- " \"\"\"\n",
+ " \n",
" return dsynonyms\n",
"\n",
"dadj = groups_for_pos('as')\n",
@@ -291,12 +424,7 @@
":\n",
" return True\n",
" # Then, print everything between .8 and .9 to see whats the best cutoff\n",
- " r = ratio(a, b)\n",
- " if .8 <= r <= .9:\n",
- " pass\n",
- " #print(a, b, r)\n",
- " return r >= .85\n",
- "\n",
+ "\"\"\"\n",
"def filter_duplicates(group):\n",
" if not group:\n",
" return []\n",
@@ -312,7 +440,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 11,
"id": "6a3c04eb-79a6-47f5-846e-93258db65921",
"metadata": {},
"outputs": [],
@@ -322,7 +450,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 63,
"id": "dfb38b21-3dc4-495a-8805-446b2e9e8483",
"metadata": {},
"outputs": [],
@@ -341,37 +469,45 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 64,
"id": "1c8175f2-817e-45ab-af0b-5ea7ee7a5dc9",
"metadata": {},
"outputs": [],
"source": [
- "corpus = dict(\n",
- " adjectives=dadj,\n",
- " nouns=dnoun,\n",
- " adverbs=dadverb,\n",
- " verbs=dverb,\n",
- " prefixes=dprefix,\n",
- " suffixes=dsuffix,\n",
- ")\n",
- "filtered_corpus = {}\n",
- "for k, d in corpus.items():\n",
- " filtered_corpus[k] = process_groups(d)"
+ "corpus = [\n",
+ " dict(name='suffixes', groups=process_groups(dsuffix), portion=.9),\n",
+ " dict(name='prefixes', groups=process_groups(dprefix), portion=.8),\n",
+ " dict(name='verbs', groups=process_groups(dverb), portion=.6),\n",
+ " dict(name='adverbs', groups=process_groups(dadverb), portion=.54),\n",
+ " dict(name='nouns', groups=process_groups(dnoun), portion=.2),\n",
+ " dict(name='adjectives', groups=process_groups(dadj), portion=0),\n",
+ "]"
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 65,
"id": "b35092c5-17db-4257-bf45-83e8c3973da4",
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[Synset('basil_thyme.n.01')]"
+ ]
+ },
+ "execution_count": 65,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
- "filtered_corpus['adverbs']"
+ "wn.synsets('satureja_acinos')"
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 66,
"id": "8025664c-e116-481a-9609-d58200f773ec",
"metadata": {},
"outputs": [],
@@ -381,15 +517,72 @@
},
{
"cell_type": "code",
+ "execution_count": 67,
+ "id": "77256802-04ef-4908-9b39-e9381f6abac5",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "(13824, 3976, 490, 7141, 226, 196)"
+ ]
+ },
+ "execution_count": 67,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(dnoun), len(dadj), len(dadverb), len(dverb), len(dprefix), len(dsuffix)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 68,
+ "id": "66256c50-fcbc-42ad-a17b-057fd0d7dea1",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "25853"
+ ]
+ },
+ "execution_count": 68,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "sum((len(dnoun), len(dadj), len(dadverb), len(dverb), len(dprefix), len(dsuffix)))"
+ ]
+ },
+ {
+ "cell_type": "code",
"execution_count": null,
- "id": "fccac4d7-af42-4445-8dd5-6f4b0d3aa9ca",
+ "id": "635b6a6c-c084-4584-a63f-cde4221e0ad9",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f48e2eec-2ab1-4398-a862-172398c413a0",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 70,
+ "id": "1f2cca38-5d1a-421e-9bcc-09d68880c6f1",
"metadata": {},
"outputs": [],
"source": [
- "\n",
"with open('../static/corpus.js', 'w') as f:\n",
" f.write('var corpus = ')\n",
- " json.dump(filtered_corpus, f)"
+ " json.dump(corpus, f)"
]
},
{
@@ -398,9 +591,7 @@
"id": "551ce71f-0d75-4e41-8387-808db1e5e20f",
"metadata": {},
"outputs": [],
- "source": [
- "similar('slow', 'slowl')"
- ]
+ "source": []
},
{
"cell_type": "code",
@@ -419,9 +610,7 @@
"output_type": "execute_result"
}
],
- "source": [
- "len(filtered_corpus)"
- ]
+ "source": []
},
{
"cell_type": "code",
@@ -440,9 +629,7 @@
"output_type": "execute_result"
}
],
- "source": [
- "len(dsuffix)"
- ]
+ "source": []
},
{
"cell_type": "code",